From 9c507f7f626dd0f499b7e7fbe2651c1af4a8d4fa Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 12 May 2018 19:09:34 +0200 Subject: [PATCH] Refactor some code to handle command line imports. --- app/Console/Commands/CreateImport.php | 204 +++++++++++++----- app/Factory/TransactionJournalFactory.php | 2 +- .../Import/JobStatusController.php | 4 +- app/Import/Routine/FakeRoutine.php | 11 +- app/Import/Routine/FileRoutine.php | 6 +- app/Import/Storage/ImportArrayStorage.php | 38 ++-- .../ImportJob/ImportJobRepository.php | 53 +++++ .../ImportJobRepositoryInterface.php | 11 + .../Journal/JournalRepository.php | 2 +- .../Import/Routine/File/MappingConverger.php | 2 +- 10 files changed, 248 insertions(+), 85 deletions(-) diff --git a/app/Console/Commands/CreateImport.php b/app/Console/Commands/CreateImport.php index d351c05b8d..0241976e6e 100644 --- a/app/Console/Commands/CreateImport.php +++ b/app/Console/Commands/CreateImport.php @@ -24,13 +24,14 @@ declare(strict_types=1); namespace FireflyIII\Console\Commands; +use Exception; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Import\Prerequisites\PrerequisitesInterface; use FireflyIII\Import\Routine\RoutineInterface; +use FireflyIII\Import\Storage\ImportArrayStorage; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Repositories\User\UserRepositoryInterface; -use FireflyIII\Services\Internal\File\EncryptService; use Illuminate\Console\Command; -use Illuminate\Support\MessageBag; use Log; use Preferences; @@ -54,18 +55,17 @@ class CreateImport extends Command */ protected $signature = 'firefly:create-import - {file : The file to import.} - {configuration : The configuration file to use for the import.} + {file? : The file to import.} + {configuration? : The configuration file to use for the import.} {--type=csv : The file type of the import.} - {--user= : The user ID that the import should import for.} + {--provider=file : The file type of the import.} + {--user=1 : The user ID that the import should import for.} {--token= : The user\'s access token.} {--start : Starts the job immediately.}'; /** * Run the command. * - * @noinspection MultipleReturnStatementsInspection - * * @throws FireflyException */ public function handle(): int @@ -76,77 +76,155 @@ class CreateImport extends Command return 1; } /** @var UserRepositoryInterface $userRepository */ - $userRepository = app(UserRepositoryInterface::class); - $file = $this->argument('file'); - $configuration = $this->argument('configuration'); - $user = $userRepository->findNull((int)$this->option('user')); - $cwd = getcwd(); - $type = strtolower($this->option('type')); + $userRepository = app(UserRepositoryInterface::class); + $file = (string)$this->argument('file'); + $configuration = (string)$this->argument('configuration'); + $user = $userRepository->findNull((int)$this->option('user')); + $cwd = getcwd(); + $type = strtolower((string)$this->option('type')); + $provider = strtolower((string)$this->option('provider')); + $configurationData = []; if (!$this->validArguments()) { $this->errorLine('Invalid arguments.'); return 1; } + if (\strlen($configuration) > 0) { + $configurationData = json_decode(file_get_contents($configuration), true); + if (null === $configurationData) { + $this->errorLine(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd)); - $configurationData = json_decode(file_get_contents($configuration), true); - if (null === $configurationData) { - $this->errorLine(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd)); - - return 1; + return 1; + } } + $this->infoLine(sprintf('Going to create a job to import file: %s', $file)); $this->infoLine(sprintf('Using configuration file: %s', $configuration)); $this->infoLine(sprintf('Import into user: #%d (%s)', $user->id, $user->email)); - $this->infoLine(sprintf('Type of import: %s', $type)); + $this->infoLine(sprintf('Type of import: %s', $provider)); /** @var ImportJobRepositoryInterface $jobRepository */ $jobRepository = app(ImportJobRepositoryInterface::class); $jobRepository->setUser($user); - $job = $jobRepository->create($type); - $this->infoLine(sprintf('Created job "%s"', $job->key)); + $importJob = $jobRepository->create($provider); + $this->infoLine(sprintf('Created job "%s"', $importJob->key)); - /** @var EncryptService $service */ - $service = app(EncryptService::class); - $service->encrypt($file, $job->key); - - $this->infoLine('Stored import data...'); - - $jobRepository->setConfiguration($job, $configurationData); - $jobRepository->updateStatus($job, 'configured'); - $this->infoLine('Stored configuration...'); - - if (true === $this->option('start')) { - $this->infoLine('The import will start in a moment. This process is not visible...'); - Log::debug('Go for import!'); - - // normally would refer to other firefly:start-import but that doesn't seem to work all to well... - - // start the actual routine: - $type = 'csv' === $job->file_type ? 'file' : $job->file_type; - $key = sprintf('import.routine.%s', $type); - $className = config($key); - if (null === $className || !class_exists($className)) { - throw new FireflyException(sprintf('Cannot find import routine class for job of type "%s".', $type)); // @codeCoverageIgnore + // make sure that job has no prerequisites. + if ((bool)config(sprintf('import.has_prereq.%s', $provider))) { + // make prerequisites thing. + $class = (string)config(sprintf('import.prerequisites.%s', $provider)); + if (!class_exists($class)) { + throw new FireflyException(sprintf('No class to handle configuration for "%s".', $provider)); // @codeCoverageIgnore } - /** @var RoutineInterface $routine */ - $routine = app($className); - $routine->setJob($job); - $routine->run(); + /** @var PrerequisitesInterface $object */ + $object = app($class); + $object->setUser($user); + if (!$object->isComplete()) { + $this->errorLine(sprintf('Import provider "%s" has prerequisites that can only be filled in using the browser.', $provider)); - // give feedback. - /** @var MessageBag $error */ - foreach ($routine->getErrors() as $index => $error) { - $this->errorLine(sprintf('Error importing line #%d: %s', $index, $error)); + return 1; } - $this->infoLine( - sprintf( - 'The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines() - ) - ); } + // store file as attachment. + if (\strlen($file) > 0) { + $messages = $jobRepository->storeCLIUpload($importJob, 'import_file', $file); + if ($messages->count() > 0) { + $this->errorLine($messages->first()); + + return 1; + } + $this->infoLine('File content saved.'); + } + + $this->infoLine('Job configuration saved.'); + $jobRepository->setConfiguration($importJob, $configurationData); + $jobRepository->setStatus($importJob, 'ready_to_run'); + + + if (true === $this->option('start')) { + $this->infoLine('The has started. The process is not visible. Please wait.'); + Log::debug('Go for import!'); + + // run it! + $key = sprintf('import.routine.%s', $provider); + $className = config($key); + if (null === $className || !class_exists($className)) { + // @codeCoverageIgnoreStart + $this->errorLine(sprintf('No routine for provider "%s"', $provider)); + + return 1; + // @codeCoverageIgnoreEnd + } + + // keep repeating this call until job lands on "provider_finished" + $valid = ['provider_finished']; + $count = 0; + while (!\in_array($importJob->status, $valid, true) && $count < 6) { + Log::debug(sprintf('Now in loop #%d.', $count + 1)); + /** @var RoutineInterface $routine */ + $routine = app($className); + $routine->setImportJob($importJob); + try { + $routine->run(); + } catch (FireflyException|Exception $e) { + $message = 'The import routine crashed: ' . $e->getMessage(); + Log::error($message); + Log::error($e->getTraceAsString()); + + // set job errored out: + $jobRepository->setStatus($importJob, 'error'); + $this->errorLine($message); + + return 1; + } + $count++; + } + if ($importJob->status === 'provider_finished') { + $this->infoLine('Import has finished. Please wait for storage of data.'); + // set job to be storing data: + $jobRepository->setStatus($importJob, 'storing_data'); + + /** @var ImportArrayStorage $storage */ + $storage = app(ImportArrayStorage::class); + $storage->setImportJob($importJob); + + try { + $storage->store(); + } catch (FireflyException|Exception $e) { + $message = 'The import routine crashed: ' . $e->getMessage(); + Log::error($message); + Log::error($e->getTraceAsString()); + + // set job errored out: + $jobRepository->setStatus($importJob, 'error'); + $this->errorLine($message); + + return 1; + } + // set storage to be finished: + $jobRepository->setStatus($importJob, 'storage_finished'); + } + + // give feedback: + $this->infoLine('Job has finished.'); + if (null !== $importJob->tag) { + $this->infoLine(sprintf('%d transaction(s) have been imported.', $importJob->tag->transactionJournals->count())); + $this->infoLine(sprintf('You can find your transactions under tag "%s"', $importJob->tag->tag)); + } + + if (null === $importJob->tag) { + $this->errorLine('No transactions have been imported :(.'); + } + if (\count($importJob->errors) > 0) { + $this->infoLine(sprintf('%d error(s) occurred:', \count($importJob->errors))); + foreach ($importJob->errors as $err) { + $this->errorLine('- ' . $err); + } + } + } // clear cache for user: Preferences::setForUser($user, 'lastActivity', microtime()); @@ -187,20 +265,28 @@ class CreateImport extends Command $cwd = getcwd(); $validTypes = config('import.options.file.import_formats'); $type = strtolower($this->option('type')); + $provider = strtolower($this->option('provider')); + $enabled = (bool)config(sprintf('import.enabled.%s', $provider)); - if (!\in_array($type, $validTypes, true)) { + if (false === $enabled) { + $this->errorLine(sprintf('Provider "%s" is not enabled.', $provider)); + + return false; + } + + if ($provider === 'file' && !\in_array($type, $validTypes, true)) { $this->errorLine(sprintf('Cannot import file of type "%s"', $type)); return false; } - if (!file_exists($file)) { + if ($provider === 'file' && !file_exists($file)) { $this->errorLine(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd)); return false; } - if (!file_exists($configuration)) { + if ($provider === 'file' && !file_exists($configuration)) { $this->errorLine(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd)); return false; diff --git a/app/Factory/TransactionJournalFactory.php b/app/Factory/TransactionJournalFactory.php index e71bb1aed5..da9e311be1 100644 --- a/app/Factory/TransactionJournalFactory.php +++ b/app/Factory/TransactionJournalFactory.php @@ -94,7 +94,7 @@ class TransactionJournalFactory // store date meta fields (if present): $fields = ['sepa-cc', 'sepa-ct-op', 'sepa-ct-id', 'sepa-db', 'sepa-country', 'sepa-ep', 'sepa-ci', 'interest_date', 'book_date', 'process_date', - 'due_date', 'payment_date', 'invoice_date', 'internal_reference', 'bunq_payment_id', 'importHash', 'external_id']; + 'due_date', 'payment_date', 'invoice_date', 'internal_reference', 'bunq_payment_id', 'importHash','importHashV2', 'external_id']; foreach ($fields as $field) { $this->storeMeta($journal, $data, $field); diff --git a/app/Http/Controllers/Import/JobStatusController.php b/app/Http/Controllers/Import/JobStatusController.php index cefd1d43ba..c6a3ae0bad 100644 --- a/app/Http/Controllers/Import/JobStatusController.php +++ b/app/Http/Controllers/Import/JobStatusController.php @@ -116,7 +116,7 @@ class JobStatusController extends Controller public function start(ImportJob $importJob): JsonResponse { // catch impossible status: - $allowed = ['ready_to_run', 'need_job_config','error','running']; + $allowed = ['ready_to_run', 'need_job_config']; // todo remove error and running. if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) { @@ -174,7 +174,7 @@ class JobStatusController extends Controller public function store(ImportJob $importJob): JsonResponse { // catch impossible status: - $allowed = ['provider_finished', 'storing_data','error']; + $allowed = ['provider_finished', 'storing_data']; if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) { Log::error('Job is not ready.'); diff --git a/app/Import/Routine/FakeRoutine.php b/app/Import/Routine/FakeRoutine.php index 32a48ccb84..a40bf40010 100644 --- a/app/Import/Routine/FakeRoutine.php +++ b/app/Import/Routine/FakeRoutine.php @@ -55,14 +55,15 @@ class FakeRoutine implements RoutineInterface public function run(): void { Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->importJob->status)); - if ($this->importJob->status !== 'running') { - throw new FireflyException('This fake job should not be started.'); // @codeCoverageIgnore + if ($this->importJob->status !== 'ready_to_run') { + throw new FireflyException(sprintf('Fake job should have status "ready_to_run", not "%s"', $this->importJob->status)); // @codeCoverageIgnore } switch ($this->importJob->stage) { default: throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->importJob->stage)); // @codeCoverageIgnore case 'new': + $this->repository->setStatus($this->importJob, 'running'); /** @var StageNewHandler $handler */ $handler = app(StageNewHandler::class); $handler->run(); @@ -72,13 +73,15 @@ class FakeRoutine implements RoutineInterface return; case 'ahoy': + $this->repository->setStatus($this->importJob, 'running'); /** @var StageAhoyHandler $handler */ $handler = app(StageAhoyHandler::class); $handler->run(); - $this->repository->setStatus($this->importJob, 'need_job_config'); + $this->repository->setStatus($this->importJob, 'ready_to_run'); $this->repository->setStage($this->importJob, 'final'); break; case 'final': + $this->repository->setStatus($this->importJob, 'running'); /** @var StageFinalHandler $handler */ $handler = app(StageFinalHandler::class); $handler->setImportJob($this->importJob); @@ -96,7 +99,7 @@ class FakeRoutine implements RoutineInterface */ public function setImportJob(ImportJob $importJob): void { - $this->importJob = $importJob; + $this->importJob = $importJob; $this->repository = app(ImportJobRepositoryInterface::class); $this->repository->setUser($importJob->user); } diff --git a/app/Import/Routine/FileRoutine.php b/app/Import/Routine/FileRoutine.php index 88e6e62a16..98de371df0 100644 --- a/app/Import/Routine/FileRoutine.php +++ b/app/Import/Routine/FileRoutine.php @@ -48,10 +48,8 @@ class FileRoutine implements RoutineInterface public function run(): void { Log::debug(sprintf('Now in run() for file routine with status: %s', $this->importJob->status)); - if ($this->importJob->status !== 'running') { - throw new FireflyException('This file import job should not be started.'); // @codeCoverageIgnore - } - if ($this->importJob->stage === 'ready_to_run') { + if ($this->importJob->status === 'ready_to_run') { + $this->repository->setStatus($this->importJob, 'running'); // get processor, depending on file type // is just CSV for now. $processor = $this->getProcessor(); diff --git a/app/Import/Storage/ImportArrayStorage.php b/app/Import/Storage/ImportArrayStorage.php index 7699caeb1f..76168ee1da 100644 --- a/app/Import/Storage/ImportArrayStorage.php +++ b/app/Import/Storage/ImportArrayStorage.php @@ -163,6 +163,22 @@ class ImportArrayStorage } + /** + * @param array $transaction + * + * @throws FireflyException + * @return string + */ + private function getHash(array $transaction): string + { + $json = json_encode($transaction); + if ($json === false) { + throw new FireflyException('Could not encode import array. Please see the logs.', $transaction); // @codeCoverageIgnore + } + + return hash('sha256', $json, false); + } + /** * Gets the users rules. * @@ -187,6 +203,7 @@ class ImportArrayStorage { /** @var JournalCollectorInterface $collector */ $collector = app(JournalCollectorInterface::class); + $collector->setUser($this->importJob->user); $collector->setAllAssetAccounts() ->setTypes([TransactionType::TRANSFER]) ->withOpposingAccount(); @@ -198,20 +215,13 @@ class ImportArrayStorage /** * Check if the hash exists for the array the user wants to import. * - * @param array $transaction + * @param string $hash * * @return int|null * @throws FireflyException */ - private function hashExists(array $transaction): ?int + private function hashExists(string $hash): ?int { - $json = json_encode($transaction); - if ($json === false) { - throw new FireflyException('Could not encode import array. Please see the logs.', $transaction); // @codeCoverageIgnore - } - $hash = hash('sha256', $json, false); - - // find it! $entry = $this->journalRepos->findByHash($hash); if (null === $entry) { return null; @@ -328,12 +338,13 @@ class ImportArrayStorage foreach ($array as $index => $transaction) { Log::debug(sprintf('Now at item %d out of %d', $index + 1, $count)); - $existingId = $this->hashExists($transaction); + $hash = $this->getHash($transaction); + $existingId = $this->hashExists($hash); if (null !== $existingId) { $this->logDuplicateObject($transaction, $existingId); $this->repository->addErrorMessage( $this->importJob, sprintf( - 'Entry #%d ("%s") could not be imported. It already exists.', + 'Row #%d ("%s") could not be imported. It already exists.', $index, $transaction['description'] ) ); @@ -344,7 +355,7 @@ class ImportArrayStorage $this->logDuplicateTransfer($transaction); $this->repository->addErrorMessage( $this->importJob, sprintf( - 'Entry #%d ("%s") could not be imported. Such a transfer already exists.', + 'Row #%d ("%s") could not be imported. Such a transfer already exists.', $index, $transaction['description'] ) @@ -352,7 +363,8 @@ class ImportArrayStorage continue; } } - $toStore[] = $transaction; + $transaction['importHashV2'] = $hash; + $toStore[] = $transaction; } $count = \count($toStore); if ($count === 0) { diff --git a/app/Repositories/ImportJob/ImportJobRepository.php b/app/Repositories/ImportJob/ImportJobRepository.php index c375c36bf1..515a001360 100644 --- a/app/Repositories/ImportJob/ImportJobRepository.php +++ b/app/Repositories/ImportJob/ImportJobRepository.php @@ -475,6 +475,59 @@ class ImportJobRepository implements ImportJobRepositoryInterface $this->user = $user; } + /** + * Handle upload for job. + * + * @param ImportJob $job + * @param string $name + * @param UploadedFile $file + * + * @return MessageBag + * @throws FireflyException + */ + public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag + { + $messages = new MessageBag; + + if (!file_exists($fileName)) { + $messages->add('notfound', sprintf('File not found: %s', $fileName)); + + return $messages; + } + + $count = $job->attachments()->get()->filter( + function (Attachment $att) use ($name) { + return $att->filename === $name; + } + )->count(); + + if ($count > 0) { + // don't upload, but also don't complain about it. + Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name)); + + return new MessageBag; + } + $content = file_get_contents($fileName); + $attachment = new Attachment; // create Attachment object. + $attachment->user()->associate($job->user); + $attachment->attachable()->associate($job); + $attachment->md5 = md5($content); + $attachment->filename = $name; + $attachment->mime = 'plain/txt'; + $attachment->size = \strlen($content); + $attachment->uploaded = 0; + $attachment->save(); + $encrypted = Crypt::encrypt($content); + + // store it: + $this->uploadDisk->put($attachment->fileName(), $encrypted); + $attachment->uploaded = 1; // update attachment + $attachment->save(); + + // return it. + return new MessageBag; + } + /** * Handle upload for job. * diff --git a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php index 8e5fba8756..baf1d51ce1 100644 --- a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php +++ b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php @@ -56,6 +56,17 @@ interface ImportJobRepositoryInterface */ public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag; + /** + * Store file. + * + * @param ImportJob $job + * @param string $name + * @param string $fileName + * + * @return MessageBag + */ + public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag; + /** * @param ImportJob $job * @param array $transactions diff --git a/app/Repositories/Journal/JournalRepository.php b/app/Repositories/Journal/JournalRepository.php index 58ca7cf4be..a1539b3c0a 100644 --- a/app/Repositories/Journal/JournalRepository.php +++ b/app/Repositories/Journal/JournalRepository.php @@ -159,7 +159,7 @@ class JournalRepository implements JournalRepositoryInterface { return TransactionJournalMeta ::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id') - ->where('data', $hash) + ->where('data', json_encode($hash)) ->where('name', 'importHashV2') ->first(['journal_meta.*']); } diff --git a/app/Support/Import/Routine/File/MappingConverger.php b/app/Support/Import/Routine/File/MappingConverger.php index d3a7af5ceb..08fd612526 100644 --- a/app/Support/Import/Routine/File/MappingConverger.php +++ b/app/Support/Import/Routine/File/MappingConverger.php @@ -168,7 +168,7 @@ class MappingConverger $newRole = 'opposing-id'; break; } - Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole)); + Log::debug(sprintf('Role was "%s", but because of mapping (mapped to #%d), role becomes "%s"', $role, $mapped, $newRole)); // also store the $mapped values in a "mappedValues" array. $this->mappedValues[$newRole][] = $mapped;