IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

Import routine can handle new SEPA fields and many new date fields. See #1248

Browse Source
This commit is contained in:
James Cole 2018-03-19 19:38:17 +01:00
parent 4e69bc0e32
commit aecffe10d9
No known key found for this signature in database
GPG Key ID: C16961E655E74B5E
6 changed files with 113 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
app/Console/Commands/CreateImport.php
View File

@ -80,7 +80,7 @@ class CreateImport extends Command
public function handle()
{
if (!$this->verifyAccessToken()) {
$this->error('Invalid access token.');
$this->errorLine('Invalid access token.');
return;
}
@ -98,39 +98,34 @@ class CreateImport extends Command
$configurationData = json_decode(file_get_contents($configuration), true);
if (null === $configurationData) {
$this->error(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd));
$this->errorLine(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd));
return;
}
$this->line(sprintf('Going to create a job to import file: %s', $file));
$this->line(sprintf('Using configuration file: %s', $configuration));
$this->line(sprintf('Import into user: #%d (%s)', $user->id, $user->email));
$this->line(sprintf('Type of import: %s', $type));
$this->infoLine(sprintf('Going to create a job to import file: %s', $file));
$this->infoLine(sprintf('Using configuration file: %s', $configuration));
$this->infoLine(sprintf('Import into user: #%d (%s)', $user->id, $user->email));
$this->infoLine(sprintf('Type of import: %s', $type));
/** @var ImportJobRepositoryInterface $jobRepository */
$jobRepository = app(ImportJobRepositoryInterface::class);
$jobRepository->setUser($user);
$job = $jobRepository->create($type);
$this->line(sprintf('Created job "%s"', $job->key));
$this->infoLine(sprintf('Created job "%s"', $job->key));
Artisan::call('firefly:encrypt-file', ['file' => $file, 'key' => $job->key]);
$this->line('Stored import data...');
$this->infoLine('Stored import data...');
$jobRepository->setConfiguration($job, $configurationData);
$jobRepository->updateStatus($job, 'configured');
$this->line('Stored configuration...');
$this->infoLine('Stored configuration...');
if (true === $this->option('start')) {
$this->line('The import will start in a moment. This process is not visible...');
$this->infoLine('The import will start in a moment. This process is not visible...');
Log::debug('Go for import!');
// normally would refer to other firefly:start-import but that doesn't seem to work all to well...
$monolog = Log::getMonolog();
$handler = new CommandHandler($this);
$formatter = new LineFormatter(null, null, false, true);
$handler->setFormatter($formatter);
$monolog->pushHandler($handler);
// start the actual routine:
$type = 'csv' === $job->file_type ? 'file' : $job->file_type;
@ -147,9 +142,9 @@ class CreateImport extends Command
// give feedback.
/** @var MessageBag $error */
foreach ($routine->getErrors() as $index => $error) {
$this->error(sprintf('Error importing line #%d: %s', $index, $error));
$this->errorLine(sprintf('Error importing line #%d: %s', $index, $error));
}
$this->line(
$this->infoLine(
sprintf(
'The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines()
)
@ -179,29 +174,50 @@ class CreateImport extends Command
$validTypes = config('import.options.file.import_formats');
$type = strtolower($this->option('type'));
if (null === $user) {
$this->error(sprintf('There is no user with ID %d.', $this->option('user')));
$this->errorLine(sprintf('There is no user with ID %d.', $this->option('user')));
return false;
}
if (!in_array($type, $validTypes)) {
$this->error(sprintf('Cannot import file of type "%s"', $type));
$this->errorLine(sprintf('Cannot import file of type "%s"', $type));
return false;
}
if (!file_exists($file)) {
$this->error(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd));
$this->errorLine(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd));
return false;
}
if (!file_exists($configuration)) {
$this->error(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd));
$this->errorLine(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd));
return false;
}
return true;
}
/**
* @param string $message
* @param array|null $data
*/
private function errorLine(string $message, array $data = null): void
{
Log::error($message, $data?? []);
$this->error($message);
}
/**
* @param string $message
* @param array $data
*/
private function infoLine(string $message, array $data = null): void
{
Log::info($message, $data?? []);
$this->line($message);
}
}

41
app/Console/Commands/Import.php
View File

@ -23,7 +23,6 @@ declare(strict_types=1);
namespace FireflyIII\Console\Commands;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\Logging\CommandHandler;
use FireflyIII\Import\Routine\RoutineInterface;
use FireflyIII\Models\ImportJob;
use Illuminate\Console\Command;
@ -68,21 +67,17 @@ class Import extends Command
$jobKey = $this->argument('key');
$job = ImportJob::where('key', $jobKey)->first();
if (null === $job) {
$this->error(sprintf('No job found with key "%s"', $jobKey));
$this->errorLine(sprintf('No job found with key "%s"', $jobKey));
return;
}
if (!$this->isValid($job)) {
Log::error('Job is not valid for some reason. Exit.');
$this->errorLine('Job is not valid for some reason. Exit.');
return;
}
$this->line(sprintf('Going to import job with key "%s" of type "%s"', $job->key, $job->file_type));
$monolog = Log::getMonolog();
$handler = new CommandHandler($this);
$monolog->pushHandler($handler);
$this->infoLine(sprintf('Going to import job with key "%s" of type "%s"', $job->key, $job->file_type));
// actually start job:
$type = 'csv' === $job->file_type ? 'file' : $job->file_type;
@ -99,16 +94,37 @@ class Import extends Command
/** @var MessageBag $error */
foreach ($routine->getErrors() as $index => $error) {
$this->error(sprintf('Error importing line #%d: %s', $index, $error));
$this->errorLine(sprintf('Error importing line #%d: %s', $index, $error));
}
$this->line(
$this->infoLine(
sprintf('The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines())
);
return;
}
/**
* @param string $message
* @param array|null $data
*/
private function errorLine(string $message, array $data = null): void
{
Log::error($message, $data ?? []);
$this->error($message);
}
/**
* @param string $message
* @param array $data
*/
private function infoLine(string $message, array $data = null): void
{
Log::info($message, $data ?? []);
$this->line($message);
}
/**
* Check if job is valid to be imported.
*
@ -119,15 +135,14 @@ class Import extends Command
private function isValid(ImportJob $job): bool
{
if (null === $job) {
Log::error('This job does not seem to exist.');
$this->error('This job does not seem to exist.');
$this->errorLine('This job does not seem to exist.');
return false;
}
if ('configured' !== $job->status) {
Log::error(sprintf('This job is not ready to be imported (status is %s).', $job->status));
$this->error('This job is not ready to be imported.');
$this->errorLine('This job is not ready to be imported.');
return false;
}

6
app/Import/Object/ImportAccount.php
View File

@ -312,9 +312,9 @@ class ImportAccount
Log::debug('Finding a mapped account based on', $array);
$search = intval($array['mapped'] ?? 0);
$account = $this->repository->find($search);
$account = $this->repository->findNull($search);
if (null === $account->id) {
if (null === $account) {
Log::error(sprintf('There is no account with id #%d. Invalid mapping will be ignored!', $search));
return null;
@ -382,7 +382,7 @@ class ImportAccount
// 4: if search for an asset account, fall back to given "default account" (mandatory)
if (AccountType::ASSET === $this->expectedType) {
$this->account = $this->repository->find($this->defaultAccountId);
$this->account = $this->repository->findNull($this->defaultAccountId);
Log::debug(sprintf('Fall back to default account #%d "%s"', $this->account->id, $this->account->name));
return true;

32
app/Import/Object/ImportJournal.php
View File

@ -54,6 +54,8 @@ class ImportJournal
public $hash;
/** @var array */
public $metaDates = [];
/** @var array */
public $metaFields = [];
/** @var string */
public $notes = '';
/** @var ImportAccount */
@ -192,6 +194,18 @@ class ImportJournal
case 'account-id':
$this->asset->setAccountId($array);
break;
case 'sepa-cc':
case 'sepa-ct-op':
case 'sepa-ct-id':
case 'sepa-db':
case 'sepa-country':
case 'sepa-ep':
case 'sepa-ci':
$value = trim(strval($array['value']));
if (strlen($value) > 0) {
$this->metaFields[$array['role']] = $value;
}
break;
case 'amount':
$this->amount = $array;
break;
@ -252,12 +266,6 @@ class ImportJournal
case 'description':
$this->description .= $array['value'];
break;
case 'sepa-ct-op':
case 'sepa-ct-id':
case 'sepa-db':
$this->notes .= ' ' . $array['value'];
$this->notes = trim($this->notes);
break;
case 'note':
$this->notes .= ' ' . $array['value'];
$this->notes = trim($this->notes);
@ -265,6 +273,9 @@ class ImportJournal
case 'external-id':
$this->externalId = $array['value'];
break;
case 'internal-reference':
$this->metaFields['internal_reference'] = $array['value'];
break;
case '_ignore':
break;
case 'ing-debit-credit':
@ -299,6 +310,15 @@ class ImportJournal
case 'date-process':
$this->metaDates['process_date'] = $array['value'];
break;
case 'date-due':
$this->metaDates['due_date'] = $array['value'];
break;
case 'date-payment':
$this->metaDates['payment_date'] = $array['value'];
break;
case 'date-invoice':
$this->metaDates['invoice_date'] = $array['value'];
break;
}
}

17
app/Import/Storage/ImportStorage.php
View File

@ -31,6 +31,7 @@ use FireflyIII\Models\Note;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
use Illuminate\Support\Collection;
use Log;
use Preferences;
@ -61,9 +62,11 @@ class ImportStorage
/** @var Collection */
protected $bills;
/** @var int */
protected $defaultCurrencyId = 1; // yes, hard coded
protected $defaultCurrencyId = 1;
/** @var ImportJob */
protected $job;
protected $job; // yes, hard coded
/** @var JournalRepositoryInterface */
protected $journalRepository;
/** @var ImportJobRepositoryInterface */
protected $repository;
/** @var Collection */
@ -104,8 +107,10 @@ class ImportStorage
*/
public function setJob(ImportJob $job)
{
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository = app(ImportJobRepositoryInterface::class);
$this->journalRepository = app(JournalRepositoryInterface::class);
$this->repository->setUser($job->user);
$this->journalRepository->setUser($job->user);
$config = $this->repository->getConfiguration($job);
$currency = app('amount')->getDefaultCurrencyByUser($job->user);
@ -232,9 +237,13 @@ class ImportStorage
$importJournal->bill->setAmount($amount);
$this->storeBill($journal, $importJournal->bill->getBill());
$this->storeMeta($journal, $importJournal->metaDates);
$this->storeMetaDates($journal, $importJournal->metaDates);
$this->storeTags($importJournal->tags, $journal);
foreach ($importJournal->metaFields as $field => $value) {
$this->journalRepository->setMetaString($journal, $field, $value);
}
// set notes for journal:
$dbNote = new Note();
$dbNote->noteable()->associate($journal);

9
app/Import/Storage/ImportSupport.php
View File

@ -39,6 +39,7 @@ use FireflyIII\Models\TransactionJournal;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use FireflyIII\TransactionRules\Processor;
use Illuminate\Database\Query\JoinClause;
@ -58,6 +59,8 @@ trait ImportSupport
protected $defaultCurrencyId = 1;
/** @var ImportJob */
protected $job;
/** @var JournalRepositoryInterface */
protected $journalRepository;
/** @var Collection */
protected $rules;
@ -441,7 +444,7 @@ trait ImportSupport
throw new FireflyException($errorText);
}
// save meta data:
$journal->setMeta('importHash', $parameters['hash']);
$this->journalRepository->setMetaString($journal, 'importHash', $parameters['hash']);
Log::debug(sprintf('Created journal with ID #%d', $journal->id));
// create transactions:
@ -472,13 +475,13 @@ trait ImportSupport
* @param TransactionJournal $journal
* @param array $dates
*/
private function storeMeta(TransactionJournal $journal, array $dates)
private function storeMetaDates(TransactionJournal $journal, array $dates)
{
// all other date fields as meta thing:
foreach ($dates as $name => $value) {
try {
$date = new Carbon($value);
$journal->setMeta($name, $date);
$this->journalRepository->setMetaDate($journal, $name, $date);
} catch (Exception $e) {
// don't care, ignore:
Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name));