Various cleanup in import.

This commit is contained in:
James Cole 2018-01-10 18:18:49 +01:00
parent 87dae6ea18
commit 91178d2604
No known key found for this signature in database
GPG Key ID: C16961E655E74B5E
10 changed files with 272 additions and 153 deletions

View File

@ -88,11 +88,9 @@ class FileConfigurator implements ConfiguratorInterface
if (is_null($this->job)) {
throw new FireflyException('Cannot call configureJob() without a job.');
}
$class = $this->getConfigurationClass();
$job = $this->job;
/** @var ConfigurationInterface $object */
$object = app($class);
$object->setJob($job);
$object = app($this->getConfigurationClass());
$object->setJob($this->job);
$result = $object->storeConfiguration($data);
$this->warning = $object->getWarningMessage();
@ -111,12 +109,9 @@ class FileConfigurator implements ConfiguratorInterface
if (is_null($this->job)) {
throw new FireflyException('Cannot call getNextData() without a job.');
}
$class = $this->getConfigurationClass();
$job = $this->job;
/** @var ConfigurationInterface $object */
$object = app($class);
$object->setJob($job);
$object = app($this->getConfigurationClass());
$object->setJob($this->job);
return $object->getData();
}

View File

@ -24,6 +24,7 @@ namespace FireflyIII\Import\Configuration;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Configuration\Spectre\HaveAccounts;
use Log;
@ -35,6 +36,9 @@ class SpectreConfigurator implements ConfiguratorInterface
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var string */
private $warning = '';
@ -51,12 +55,14 @@ class SpectreConfigurator implements ConfiguratorInterface
* @param array $data
*
* @return bool
* @throws FireflyException
*/
public function configureJob(array $data): bool
{
$config = $this->job->configuration;
$stage = $config['stage'];
$status = $this->job->status;
if (is_null($this->job)) {
throw new FireflyException('Cannot call configureJob() without a job.');
}
$stage = $this->getConfig()['stage'] ?? 'initial';
Log::debug(sprintf('in getNextData(), for stage "%s".', $stage));
switch ($stage) {
case 'have-accounts':
@ -66,11 +72,10 @@ class SpectreConfigurator implements ConfiguratorInterface
$class->storeConfiguration($data);
// update job for next step and set to "configured".
$config = $this->job->configuration;
$config['stage'] = 'have-account-mapping';
$this->job->configuration = $config;
$this->job->status = 'configured';
$this->job->save();
$config = $this->getConfig();
$config['stage'] = 'have-account-mapping';
$this->repository->setConfiguration($this->job, $config);
return true;
break;
default:
@ -83,12 +88,14 @@ class SpectreConfigurator implements ConfiguratorInterface
* Return the data required for the next step in the job configuration.
*
* @return array
* @throws FireflyException
*/
public function getNextData(): array
{
$config = $this->job->configuration;
$stage = $config['stage'];
$status = $this->job->status;
if (is_null($this->job)) {
throw new FireflyException('Cannot call configureJob() without a job.');
}
$stage = $this->getConfig()['stage'] ?? 'initial';
Log::debug(sprintf('in getNextData(), for stage "%s".', $stage));
switch ($stage) {
case 'has-token':
@ -96,6 +103,12 @@ class SpectreConfigurator implements ConfiguratorInterface
$config['is-redirected'] = true;
$config['stage'] = 'user-logged-in';
$status = 'configured';
// update config and status:
$this->repository->setConfiguration($this->job, $config);
$this->repository->setStatus($this->job, $status);
return $this->repository->getConfiguration($this->job);
break;
case 'have-accounts':
// use special class:
@ -108,29 +121,25 @@ class SpectreConfigurator implements ConfiguratorInterface
default:
return [];
break;
}
// update config and status:
$this->job->configuration = $config;
$this->job->status = $status;
$this->job->save();
return $this->job->configuration;
}
/**
* @return string
* @throws FireflyException
*/
public function getNextView(): string
{
$config = $this->job->configuration;
$stage = $config['stage'];
if (is_null($this->job)) {
throw new FireflyException('Cannot call configureJob() without a job.');
}
$stage = $this->getConfig()['stage'] ?? 'initial';
Log::debug(sprintf('in getNextView(), for stage "%s".', $stage));
switch ($stage) {
case 'has-token':
// redirect to Spectre.
Log::info('User is being redirected to Spectre.');
return 'import.spectre.redirect';
break;
case 'have-accounts':
@ -155,11 +164,14 @@ class SpectreConfigurator implements ConfiguratorInterface
/**
* @return bool
* @throws FireflyException
*/
public function isJobConfigured(): bool
{
$config = $this->job->configuration;
$stage = $config['stage'];
if (is_null($this->job)) {
throw new FireflyException('Cannot call configureJob() without a job.');
}
$stage = $this->getConfig()['stage'] ?? 'initial';
Log::debug(sprintf('in isJobConfigured(), for stage "%s".', $stage));
switch ($stage) {
case 'has-token':
@ -177,9 +189,14 @@ class SpectreConfigurator implements ConfiguratorInterface
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job)
public function setJob(ImportJob $job): void
{
$defaultConfig = [
// make repository
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
// set default config:
$defaultConfig = [
'has-token' => false,
'token' => '',
'token-expires' => 0,
@ -191,16 +208,31 @@ class SpectreConfigurator implements ConfiguratorInterface
'accounts' => '',
'accounts-mapped' => '',
'auto-start' => true,
'apply-rules' => true,
'match-bills' => false,
];
$extendedStatus = $job->extended_status;
$currentConfig = $this->repository->getConfiguration($job);
$finalConfig = array_merge($defaultConfig, $currentConfig);
// set default extended status:
$extendedStatus = $this->repository->getExtendedStatus($job);
$extendedStatus['steps'] = 100;
$config = $job->configuration;
$finalConfig = array_merge($defaultConfig, $config);
$job->configuration = $finalConfig;
$job->extended_status = $extendedStatus;
$job->save();
// save to job:
$job = $this->repository->setConfiguration($job, $finalConfig);
$job = $this->repository->setExtendedStatus($job, $extendedStatus);
$this->job = $job;
return;
}
/**
* Shorthand method.
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
}

View File

@ -98,7 +98,7 @@ class CsvProcessor implements FileProcessorInterface
if ($this->rowAlreadyImported($row)) {
$message = sprintf('Row #%d has already been imported.', $index);
$this->repository->addStepsDone($this->job, 5);
$this->addError($index, $message);
$this->repository->addError($this->job, $index, $message);
Log::info($message);
return null;
@ -154,23 +154,6 @@ class CsvProcessor implements FileProcessorInterface
return $this;
}
/**
* Shorthand method.
*
* @codeCoverageIgnore
*
* @param int $index
* @param string $message
*/
private function addError(int $index, string $message): void
{
$extended = $this->getExtendedStatus();
$extended['errors'][$index][] = $message;
$this->setExtendedStatus($extended);
return;
}
/**
* Add meta data to the individual value and verify that it can be handled in a later stage.
*
@ -375,7 +358,7 @@ class CsvProcessor implements FileProcessorInterface
*/
private function specifics(array $row): array
{
$config = $this->job->configuration;
$config = $this->getConfig();
$names = array_keys($config['specifics'] ?? []);
foreach ($names as $name) {
if (!in_array($name, $this->validSpecifics)) {

View File

@ -28,6 +28,7 @@ use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use Illuminate\Support\Collection;
use Log;
@ -46,6 +47,9 @@ class FileRoutine implements RoutineInterface
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* ImportRoutine constructor.
*/
@ -84,8 +88,8 @@ class FileRoutine implements RoutineInterface
*/
public function run(): bool
{
if ('configured' !== $this->job->status) {
Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->job->status));
if ('configured' !== $this->getStatus()) {
Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
return false;
}
@ -102,8 +106,7 @@ class FileRoutine implements RoutineInterface
Log::debug('Back in run()');
// update job:
$this->job->status = 'finished';
$this->job->save();
$this->setStatus('finished');
Log::debug('Updated job...');
Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
@ -125,7 +128,9 @@ class FileRoutine implements RoutineInterface
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
@ -134,18 +139,16 @@ class FileRoutine implements RoutineInterface
protected function getImportObjects(): Collection
{
$objects = new Collection;
$config = $this->job->configuration;
$fileType = $config['file-type'] ?? 'csv';
$fileType = $this->getConfig()['file-type'] ?? 'csv';
// will only respond to "file"
$class = config(sprintf('import.options.file.processors.%s', $fileType));
/** @var FileProcessorInterface $processor */
$processor = app($class);
$processor->setJob($this->job);
if ('configured' === $this->job->status) {
if ('configured' === $this->getStatus()) {
// set job as "running"...
$this->job->status = 'running';
$this->job->save();
$this->setStatus('running');
Log::debug('Job is configured, start with run()');
$processor->run();
@ -171,7 +174,7 @@ class FileRoutine implements RoutineInterface
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->job->user);
$data = [
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
'date' => new Carbon,
'description' => null,
@ -180,11 +183,10 @@ class FileRoutine implements RoutineInterface
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
$extended = $this->job->extended_status;
$extended['tag'] = $tag->id;
$this->job->extended_status = $extended;
$this->job->save();
$tag = $repository->store($data);
$extended = $this->getExtendedStatus();
$extended['tag'] = $tag->id;
$this->setExtendedStatus($extended);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
@ -199,6 +201,54 @@ class FileRoutine implements RoutineInterface
return $tag;
}
/**
* Shorthand method
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @return array
*/
private function getExtendedStatus(): array
{
return $this->repository->getExtendedStatus($this->job);
}
/**
* Shorthand method.
*
* @return string
*/
private function getStatus(): string
{
return $this->repository->getStatus($this->job);
}
/**
* @param array $extended
*/
private function setExtendedStatus(array $extended): void
{
$this->repository->setExtendedStatus($this->job, $extended);
return;
}
/**
* Shorthand
*
* @param string $status
*/
private function setStatus(string $status): void
{
$this->repository->setStatus($this->job, $status);
}
/**
* @param Collection $objects
*
@ -206,9 +256,10 @@ class FileRoutine implements RoutineInterface
*/
private function storeObjects(Collection $objects): ImportStorage
{
$config = $this->getConfig();
$storage = new ImportStorage;
$storage->setJob($this->job);
$storage->setDateFormat($this->job->configuration['date-format']);
$storage->setDateFormat($config['date-format']);
$storage->setObjects($objects);
$storage->store();
Log::info('Back in storeObjects()');

View File

@ -122,15 +122,14 @@ class SpectreRoutine implements RoutineInterface
*/
public function run(): bool
{
if ('configured' === $this->job->status) {
if ('configured' === $this->getStatus()) {
$this->repository->updateStatus($this->job, 'running');
}
Log::info(sprintf('Start with import job %s using Spectre.', $this->job->key));
set_time_limit(0);
// check if job has token first!
$config = $this->job->configuration;
$stage = $config['stage'];
$stage = $this->getConfig()['stage'] ?? 'unknown';
switch ($stage) {
case 'initial':
@ -205,21 +204,20 @@ class SpectreRoutine implements RoutineInterface
*/
protected function getCustomer(): Customer
{
$config = $this->job->configuration;
$config = $this->getConfig();
if (!is_null($config['customer'])) {
$customer = new Customer($config['customer']);
return $customer;
}
$customer = $this->createCustomer();
$config['customer'] = [
$customer = $this->createCustomer();
$config['customer'] = [
'id' => $customer->getId(),
'identifier' => $customer->getIdentifier(),
'secret' => $customer->getSecret(),
];
$this->job->configuration = $config;
$this->job->save();
$this->setConfig($config);
return $customer;
}
@ -268,19 +266,18 @@ class SpectreRoutine implements RoutineInterface
$this->repository->addStepsDone($this->job, 2);
// update job, give it the token:
$config = $this->job->configuration;
$config['has-token'] = true;
$config['token'] = $token->getToken();
$config['token-expires'] = $token->getExpiresAt()->format('U');
$config['token-url'] = $token->getConnectUrl();
$config['stage'] = 'has-token';
$this->job->configuration = $config;
$config = $this->getConfig();
$config['has-token'] = true;
$config['token'] = $token->getToken();
$config['token-expires'] = $token->getExpiresAt()->format('U');
$config['token-url'] = $token->getConnectUrl();
$config['stage'] = 'has-token';
$this->setConfig($config);
Log::debug('Job config is now', $config);
// update job, set status to "configuring".
$this->job->status = 'configuring';
$this->job->save();
$this->setStatus('configuring');
Log::debug(sprintf('Job status is now %s', $this->job->status));
}
@ -319,7 +316,8 @@ class SpectreRoutine implements RoutineInterface
$this->repository->addError($this->job, 0, 'Spectre connection failed. Did you use invalid credentials, press Cancel or failed the 2FA challenge?');
$this->repository->setTotalSteps($this->job, 1);
$this->repository->setStepsDone($this->job, 1);
$this->repository->setStatus($this->job,'error');
$this->repository->setStatus($this->job, 'error');
return;
}
@ -343,13 +341,13 @@ class SpectreRoutine implements RoutineInterface
}
// update job:
$config = $this->job->configuration;
$config['accounts'] = $all;
$config['login'] = $login->toArray();
$config['stage'] = 'have-accounts';
$this->job->configuration = $config;
$this->job->status = 'configuring';
$this->job->save();
$config = $this->getConfig();
$config['accounts'] = $all;
$config['login'] = $login->toArray();
$config['stage'] = 'have-accounts';
$this->setConfig($config);
$this->setStatus('configuring');
// add some steps done
$this->repository->addStepsDone($this->job, 2);
@ -357,6 +355,32 @@ class SpectreRoutine implements RoutineInterface
return;
}
/**
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @return array
*/
private function getExtendedStatus(): array
{
return $this->repository->getExtendedStatus($this->job);
}
/**
* Shorthand method.
*
* @return string
*/
private function getStatus(): string
{
return $this->repository->getStatus($this->job);
}
/**
* @param array $all
*
@ -433,7 +457,7 @@ class SpectreRoutine implements RoutineInterface
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->job->user);
$data = [
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
'date' => new Carbon,
'description' => null,
@ -442,11 +466,10 @@ class SpectreRoutine implements RoutineInterface
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
$extended = $this->job->extended_status;
$extended['tag'] = $tag->id;
$this->job->extended_status = $extended;
$this->job->save();
$tag = $repository->store($data);
$extended = $this->getExtendedStatus();
$extended['tag'] = $tag->id;
$this->setExtendedStatus($extended);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
@ -460,8 +483,7 @@ class SpectreRoutine implements RoutineInterface
// set status to "finished"?
// update job:
$this->job->status = 'finished';
$this->job->save();
$this->setStatus('finished');
return;
}
@ -472,8 +494,7 @@ class SpectreRoutine implements RoutineInterface
*/
private function runStageHaveMapping()
{
$config = $this->job->configuration;
$accounts = $config['accounts'] ?? [];
$accounts = $this->getConfig()['accounts'] ?? [];
$all = [];
$count = 0;
/** @var array $accountArray */
@ -508,4 +529,36 @@ class SpectreRoutine implements RoutineInterface
$this->importTransactions($all);
}
/**
* Shorthand.
*
* @param array $config
*/
private function setConfig(array $config): void
{
$this->repository->setConfiguration($this->job, $config);
return;
}
/**
* @param array $extended
*/
private function setExtendedStatus(array $extended): void
{
$this->repository->setExtendedStatus($this->job, $extended);
return;
}
/**
* Shorthand.
*
* @param string $status
*/
private function setStatus(string $status): void
{
$this->repository->setStatus($this->job, $status);
}
}

View File

@ -30,6 +30,7 @@ use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Note;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Support\Collection;
use Log;
@ -53,6 +54,8 @@ class ImportStorage
protected $defaultCurrencyId = 1;
/** @var ImportJob */
protected $job;
/** @var ImportJobRepositoryInterface */
protected $repository;
/** @var Collection */
protected $rules;
/** @var bool */
@ -63,7 +66,8 @@ class ImportStorage
private $matchBills = false;
/** @var Collection */
private $objects;
private $total = 0;
/** @var int */
private $total = 0;
/** @var array */
private $transfers = [];
@ -90,13 +94,17 @@ class ImportStorage
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
$config = $this->repository->getConfiguration($job);
$currency = app('amount')->getDefaultCurrencyByUser($this->job->user);
$this->defaultCurrencyId = $currency->id;
$this->transfers = $this->getTransfers();
$config = $job->configuration;
$this->applyRules = $config['apply-rules'] ?? false;
$this->matchBills = $config['match-bills'] ?? false;
if (true === $this->applyRules) {
Log::debug('applyRules seems to be true, get the rules.');
$this->rules = $this->getRules();
@ -109,6 +117,10 @@ class ImportStorage
}
Log::debug(sprintf('Value of apply rules is %s', var_export($this->applyRules, true)));
Log::debug(sprintf('Value of match bills is %s', var_export($this->matchBills, true)));
$this->job = $job;
}
/**
@ -152,7 +164,7 @@ class ImportStorage
*/
protected function storeImportJournal(int $index, ImportJournal $importJournal): bool
{
Log::debug(sprintf('Going to store object #%d/%d with description "%s"', ($index+1), $this->total, $importJournal->getDescription()));
Log::debug(sprintf('Going to store object #%d/%d with description "%s"', ($index + 1), $this->total, $importJournal->getDescription()));
$assetAccount = $importJournal->asset->getAccount();
$amount = $importJournal->getAmount();
$currencyId = $this->getCurrencyId($importJournal);
@ -163,7 +175,7 @@ class ImportStorage
$description = $importJournal->getDescription();
// First step done!
$this->job->addStepsDone(1);
$this->repository->addStepsDone($this->job, 1);
/**
* Check for double transfer.
@ -177,7 +189,7 @@ class ImportStorage
'opposing' => $opposingAccount->name,
];
if ($this->isDoubleTransfer($parameters) || $this->hashAlreadyImported($importJournal->hash)) {
$this->job->addStepsDone(3);
$this->repository->addStepsDone($this->job, 3);
// throw error
$message = sprintf('Detected a possible duplicate, skip this one (hash: %s).', $importJournal->hash);
Log::error($message, $parameters);
@ -201,7 +213,7 @@ class ImportStorage
unset($parameters);
// Another step done!
$this->job->addStepsDone(1);
$this->repository->addStepsDone($this->job, 1);
// store meta object things:
$this->storeCategory($journal, $importJournal->category->getCategory());
@ -225,7 +237,7 @@ class ImportStorage
$journal->save();
// Another step done!
$this->job->addStepsDone(1);
$this->repository->addStepsDone($this->job, 1);
// run rules if config calls for it:
if (true === $this->applyRules) {
@ -247,7 +259,7 @@ class ImportStorage
}
// Another step done!
$this->job->addStepsDone(1);
$this->repository->addStepsDone($this->job, 1);
$this->journals->push($journal);
Log::info(sprintf('Imported new journal #%d: "%s", amount %s %s.', $journal->id, $journal->description, $journal->transactionCurrency->code, $amount));

View File

@ -436,7 +436,7 @@ trait ImportSupport
if (!$journal->save()) {
$errorText = join(', ', $journal->getErrors()->all());
// add three steps:
$this->job->addStepsDone(3);
$this->repository->addStepsDone($this->job, 3);
// throw error
throw new FireflyException($errorText);
}

View File

@ -82,33 +82,6 @@ class ImportJob extends Model
throw new NotFoundHttpException;
}
/**
* @param int $index
* @param string $message
*
* @return bool
*/
public function addError(int $index, string $message): bool
{
$extended = $this->extended_status;
$extended['errors'][$index][] = $message;
$this->extended_status = $extended;
return true;
}
/**
* @param int $count
*/
public function addStepsDone(int $count)
{
$status = $this->extended_status;
$status['done'] += $count;
$this->extended_status = $status;
$this->save();
Log::debug(sprintf('Add %d to steps done for job "%s" making steps done %d', $count, $this->key, $status['done']));
}
/**
* @param int $count
*/

View File

@ -51,9 +51,10 @@ class ImportJobRepository implements ImportJobRepositoryInterface
*/
public function addError(ImportJob $job, int $index, string $error): ImportJob
{
$job->addError($index, $error);
$extended = $this->getExtendedStatus($job);
$extended['errors'][$index][] = $error;
return $job;
return $this->setExtendedStatus($job, $extended);
}
/**
@ -64,9 +65,11 @@ class ImportJobRepository implements ImportJobRepositoryInterface
*/
public function addStepsDone(ImportJob $job, int $steps = 1): ImportJob
{
$job->addStepsDone($steps);
$status = $this->getExtendedStatus($job);
$status['done'] += $steps;
Log::debug(sprintf('Add %d to steps done for job "%s" making steps done %d', $steps, $job->key, $status['done']));
return $job;
return $this->setExtendedStatus($status);
}
/**
@ -175,6 +178,16 @@ class ImportJobRepository implements ImportJobRepositoryInterface
return [];
}
/**
* @param ImportJob $job
*
* @return string
*/
public function getStatus(ImportJob $job): string
{
return $job->status;
}
/**
* @param ImportJob $job
* @param UploadedFile $file

View File

@ -90,6 +90,13 @@ interface ImportJobRepositoryInterface
*/
public function getExtendedStatus(ImportJob $job): array;
/**
* @param ImportJob $job
*
* @return string
*/
public function getStatus(ImportJob $job);
/**
* @param ImportJob $job
* @param UploadedFile $file