mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-01-27 08:46:40 -06:00
Refactor some code to handle command line imports.
This commit is contained in:
James Cole
parent
07da2fdda3
commit
9c507f7f62
@ -24,13 +24,14 @@ declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Console\Commands;
|
||||
|
||||
use Exception;
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Import\Prerequisites\PrerequisitesInterface;
|
||||
use FireflyIII\Import\Routine\RoutineInterface;
|
||||
use FireflyIII\Import\Storage\ImportArrayStorage;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use FireflyIII\Repositories\User\UserRepositoryInterface;
|
||||
use FireflyIII\Services\Internal\File\EncryptService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use Log;
|
||||
use Preferences;
|
||||
|
||||
@ -54,18 +55,17 @@ class CreateImport extends Command
|
||||
*/
|
||||
protected $signature
|
||||
= 'firefly:create-import
|
||||
{file : The file to import.}
|
||||
{configuration : The configuration file to use for the import.}
|
||||
{file? : The file to import.}
|
||||
{configuration? : The configuration file to use for the import.}
|
||||
{--type=csv : The file type of the import.}
|
||||
{--user= : The user ID that the import should import for.}
|
||||
{--provider=file : The file type of the import.}
|
||||
{--user=1 : The user ID that the import should import for.}
|
||||
{--token= : The user\'s access token.}
|
||||
{--start : Starts the job immediately.}';
|
||||
|
||||
/**
|
||||
* Run the command.
|
||||
*
|
||||
* @noinspection MultipleReturnStatementsInspection
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function handle(): int
|
||||
@ -77,76 +77,154 @@ class CreateImport extends Command
|
||||
}
|
||||
/** @var UserRepositoryInterface $userRepository */
|
||||
$userRepository = app(UserRepositoryInterface::class);
|
||||
$file = $this->argument('file');
|
||||
$configuration = $this->argument('configuration');
|
||||
$file = (string)$this->argument('file');
|
||||
$configuration = (string)$this->argument('configuration');
|
||||
$user = $userRepository->findNull((int)$this->option('user'));
|
||||
$cwd = getcwd();
|
||||
$type = strtolower($this->option('type'));
|
||||
$type = strtolower((string)$this->option('type'));
|
||||
$provider = strtolower((string)$this->option('provider'));
|
||||
$configurationData = [];
|
||||
|
||||
if (!$this->validArguments()) {
|
||||
$this->errorLine('Invalid arguments.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (\strlen($configuration) > 0) {
|
||||
$configurationData = json_decode(file_get_contents($configuration), true);
|
||||
if (null === $configurationData) {
|
||||
$this->errorLine(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd));
|
||||
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->infoLine(sprintf('Going to create a job to import file: %s', $file));
|
||||
$this->infoLine(sprintf('Using configuration file: %s', $configuration));
|
||||
$this->infoLine(sprintf('Import into user: #%d (%s)', $user->id, $user->email));
|
||||
$this->infoLine(sprintf('Type of import: %s', $type));
|
||||
$this->infoLine(sprintf('Type of import: %s', $provider));
|
||||
|
||||
/** @var ImportJobRepositoryInterface $jobRepository */
|
||||
$jobRepository = app(ImportJobRepositoryInterface::class);
|
||||
$jobRepository->setUser($user);
|
||||
$job = $jobRepository->create($type);
|
||||
$this->infoLine(sprintf('Created job "%s"', $job->key));
|
||||
$importJob = $jobRepository->create($provider);
|
||||
$this->infoLine(sprintf('Created job "%s"', $importJob->key));
|
||||
|
||||
/** @var EncryptService $service */
|
||||
$service = app(EncryptService::class);
|
||||
$service->encrypt($file, $job->key);
|
||||
// make sure that job has no prerequisites.
|
||||
if ((bool)config(sprintf('import.has_prereq.%s', $provider))) {
|
||||
// make prerequisites thing.
|
||||
$class = (string)config(sprintf('import.prerequisites.%s', $provider));
|
||||
if (!class_exists($class)) {
|
||||
throw new FireflyException(sprintf('No class to handle configuration for "%s".', $provider)); // @codeCoverageIgnore
|
||||
}
|
||||
/** @var PrerequisitesInterface $object */
|
||||
$object = app($class);
|
||||
$object->setUser($user);
|
||||
if (!$object->isComplete()) {
|
||||
$this->errorLine(sprintf('Import provider "%s" has prerequisites that can only be filled in using the browser.', $provider));
|
||||
|
||||
$this->infoLine('Stored import data...');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// store file as attachment.
|
||||
if (\strlen($file) > 0) {
|
||||
$messages = $jobRepository->storeCLIUpload($importJob, 'import_file', $file);
|
||||
if ($messages->count() > 0) {
|
||||
$this->errorLine($messages->first());
|
||||
|
||||
return 1;
|
||||
}
|
||||
$this->infoLine('File content saved.');
|
||||
}
|
||||
|
||||
$this->infoLine('Job configuration saved.');
|
||||
$jobRepository->setConfiguration($importJob, $configurationData);
|
||||
$jobRepository->setStatus($importJob, 'ready_to_run');
|
||||
|
||||
$jobRepository->setConfiguration($job, $configurationData);
|
||||
$jobRepository->updateStatus($job, 'configured');
|
||||
$this->infoLine('Stored configuration...');
|
||||
|
||||
if (true === $this->option('start')) {
|
||||
$this->infoLine('The import will start in a moment. This process is not visible...');
|
||||
$this->infoLine('The has started. The process is not visible. Please wait.');
|
||||
Log::debug('Go for import!');
|
||||
|
||||
// normally would refer to other firefly:start-import but that doesn't seem to work all to well...
|
||||
|
||||
// start the actual routine:
|
||||
$type = 'csv' === $job->file_type ? 'file' : $job->file_type;
|
||||
$key = sprintf('import.routine.%s', $type);
|
||||
// run it!
|
||||
$key = sprintf('import.routine.%s', $provider);
|
||||
$className = config($key);
|
||||
if (null === $className || !class_exists($className)) {
|
||||
throw new FireflyException(sprintf('Cannot find import routine class for job of type "%s".', $type)); // @codeCoverageIgnore
|
||||
// @codeCoverageIgnoreStart
|
||||
$this->errorLine(sprintf('No routine for provider "%s"', $provider));
|
||||
|
||||
return 1;
|
||||
// @codeCoverageIgnoreEnd
|
||||
}
|
||||
|
||||
// keep repeating this call until job lands on "provider_finished"
|
||||
$valid = ['provider_finished'];
|
||||
$count = 0;
|
||||
while (!\in_array($importJob->status, $valid, true) && $count < 6) {
|
||||
Log::debug(sprintf('Now in loop #%d.', $count + 1));
|
||||
/** @var RoutineInterface $routine */
|
||||
$routine = app($className);
|
||||
$routine->setJob($job);
|
||||
$routine->setImportJob($importJob);
|
||||
try {
|
||||
$routine->run();
|
||||
} catch (FireflyException|Exception $e) {
|
||||
$message = 'The import routine crashed: ' . $e->getMessage();
|
||||
Log::error($message);
|
||||
Log::error($e->getTraceAsString());
|
||||
|
||||
// give feedback.
|
||||
/** @var MessageBag $error */
|
||||
foreach ($routine->getErrors() as $index => $error) {
|
||||
$this->errorLine(sprintf('Error importing line #%d: %s', $index, $error));
|
||||
// set job errored out:
|
||||
$jobRepository->setStatus($importJob, 'error');
|
||||
$this->errorLine($message);
|
||||
|
||||
return 1;
|
||||
}
|
||||
$this->infoLine(
|
||||
sprintf(
|
||||
'The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines()
|
||||
)
|
||||
);
|
||||
$count++;
|
||||
}
|
||||
if ($importJob->status === 'provider_finished') {
|
||||
$this->infoLine('Import has finished. Please wait for storage of data.');
|
||||
// set job to be storing data:
|
||||
$jobRepository->setStatus($importJob, 'storing_data');
|
||||
|
||||
/** @var ImportArrayStorage $storage */
|
||||
$storage = app(ImportArrayStorage::class);
|
||||
$storage->setImportJob($importJob);
|
||||
|
||||
try {
|
||||
$storage->store();
|
||||
} catch (FireflyException|Exception $e) {
|
||||
$message = 'The import routine crashed: ' . $e->getMessage();
|
||||
Log::error($message);
|
||||
Log::error($e->getTraceAsString());
|
||||
|
||||
// set job errored out:
|
||||
$jobRepository->setStatus($importJob, 'error');
|
||||
$this->errorLine($message);
|
||||
|
||||
return 1;
|
||||
}
|
||||
// set storage to be finished:
|
||||
$jobRepository->setStatus($importJob, 'storage_finished');
|
||||
}
|
||||
|
||||
// give feedback:
|
||||
$this->infoLine('Job has finished.');
|
||||
if (null !== $importJob->tag) {
|
||||
$this->infoLine(sprintf('%d transaction(s) have been imported.', $importJob->tag->transactionJournals->count()));
|
||||
$this->infoLine(sprintf('You can find your transactions under tag "%s"', $importJob->tag->tag));
|
||||
}
|
||||
|
||||
if (null === $importJob->tag) {
|
||||
$this->errorLine('No transactions have been imported :(.');
|
||||
}
|
||||
if (\count($importJob->errors) > 0) {
|
||||
$this->infoLine(sprintf('%d error(s) occurred:', \count($importJob->errors)));
|
||||
foreach ($importJob->errors as $err) {
|
||||
$this->errorLine('- ' . $err);
|
||||
}
|
||||
}
|
||||
}
|
||||
// clear cache for user:
|
||||
Preferences::setForUser($user, 'lastActivity', microtime());
|
||||
|
||||
@ -187,20 +265,28 @@ class CreateImport extends Command
|
||||
$cwd = getcwd();
|
||||
$validTypes = config('import.options.file.import_formats');
|
||||
$type = strtolower($this->option('type'));
|
||||
$provider = strtolower($this->option('provider'));
|
||||
$enabled = (bool)config(sprintf('import.enabled.%s', $provider));
|
||||
|
||||
if (!\in_array($type, $validTypes, true)) {
|
||||
if (false === $enabled) {
|
||||
$this->errorLine(sprintf('Provider "%s" is not enabled.', $provider));
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($provider === 'file' && !\in_array($type, $validTypes, true)) {
|
||||
$this->errorLine(sprintf('Cannot import file of type "%s"', $type));
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!file_exists($file)) {
|
||||
if ($provider === 'file' && !file_exists($file)) {
|
||||
$this->errorLine(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd));
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!file_exists($configuration)) {
|
||||
if ($provider === 'file' && !file_exists($configuration)) {
|
||||
$this->errorLine(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd));
|
||||
|
||||
return false;
|
||||
|
||||
@ -94,7 +94,7 @@ class TransactionJournalFactory
|
||||
|
||||
// store date meta fields (if present):
|
||||
$fields = ['sepa-cc', 'sepa-ct-op', 'sepa-ct-id', 'sepa-db', 'sepa-country', 'sepa-ep', 'sepa-ci', 'interest_date', 'book_date', 'process_date',
|
||||
'due_date', 'payment_date', 'invoice_date', 'internal_reference', 'bunq_payment_id', 'importHash', 'external_id'];
|
||||
'due_date', 'payment_date', 'invoice_date', 'internal_reference', 'bunq_payment_id', 'importHash','importHashV2', 'external_id'];
|
||||
|
||||
foreach ($fields as $field) {
|
||||
$this->storeMeta($journal, $data, $field);
|
||||
|
||||
@ -116,7 +116,7 @@ class JobStatusController extends Controller
|
||||
public function start(ImportJob $importJob): JsonResponse
|
||||
{
|
||||
// catch impossible status:
|
||||
$allowed = ['ready_to_run', 'need_job_config','error','running'];
|
||||
$allowed = ['ready_to_run', 'need_job_config'];
|
||||
// todo remove error and running.
|
||||
|
||||
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
|
||||
@ -174,7 +174,7 @@ class JobStatusController extends Controller
|
||||
public function store(ImportJob $importJob): JsonResponse
|
||||
{
|
||||
// catch impossible status:
|
||||
$allowed = ['provider_finished', 'storing_data','error'];
|
||||
$allowed = ['provider_finished', 'storing_data'];
|
||||
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
|
||||
Log::error('Job is not ready.');
|
||||
|
||||
|
||||
@ -55,14 +55,15 @@ class FakeRoutine implements RoutineInterface
|
||||
public function run(): void
|
||||
{
|
||||
Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->importJob->status));
|
||||
if ($this->importJob->status !== 'running') {
|
||||
throw new FireflyException('This fake job should not be started.'); // @codeCoverageIgnore
|
||||
if ($this->importJob->status !== 'ready_to_run') {
|
||||
throw new FireflyException(sprintf('Fake job should have status "ready_to_run", not "%s"', $this->importJob->status)); // @codeCoverageIgnore
|
||||
}
|
||||
|
||||
switch ($this->importJob->stage) {
|
||||
default:
|
||||
throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->importJob->stage)); // @codeCoverageIgnore
|
||||
case 'new':
|
||||
$this->repository->setStatus($this->importJob, 'running');
|
||||
/** @var StageNewHandler $handler */
|
||||
$handler = app(StageNewHandler::class);
|
||||
$handler->run();
|
||||
@ -72,13 +73,15 @@ class FakeRoutine implements RoutineInterface
|
||||
|
||||
return;
|
||||
case 'ahoy':
|
||||
$this->repository->setStatus($this->importJob, 'running');
|
||||
/** @var StageAhoyHandler $handler */
|
||||
$handler = app(StageAhoyHandler::class);
|
||||
$handler->run();
|
||||
$this->repository->setStatus($this->importJob, 'need_job_config');
|
||||
$this->repository->setStatus($this->importJob, 'ready_to_run');
|
||||
$this->repository->setStage($this->importJob, 'final');
|
||||
break;
|
||||
case 'final':
|
||||
$this->repository->setStatus($this->importJob, 'running');
|
||||
/** @var StageFinalHandler $handler */
|
||||
$handler = app(StageFinalHandler::class);
|
||||
$handler->setImportJob($this->importJob);
|
||||
|
||||
@ -48,10 +48,8 @@ class FileRoutine implements RoutineInterface
|
||||
public function run(): void
|
||||
{
|
||||
Log::debug(sprintf('Now in run() for file routine with status: %s', $this->importJob->status));
|
||||
if ($this->importJob->status !== 'running') {
|
||||
throw new FireflyException('This file import job should not be started.'); // @codeCoverageIgnore
|
||||
}
|
||||
if ($this->importJob->stage === 'ready_to_run') {
|
||||
if ($this->importJob->status === 'ready_to_run') {
|
||||
$this->repository->setStatus($this->importJob, 'running');
|
||||
// get processor, depending on file type
|
||||
// is just CSV for now.
|
||||
$processor = $this->getProcessor();
|
||||
|
||||
@ -163,6 +163,22 @@ class ImportArrayStorage
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $transaction
|
||||
*
|
||||
* @throws FireflyException
|
||||
* @return string
|
||||
*/
|
||||
private function getHash(array $transaction): string
|
||||
{
|
||||
$json = json_encode($transaction);
|
||||
if ($json === false) {
|
||||
throw new FireflyException('Could not encode import array. Please see the logs.', $transaction); // @codeCoverageIgnore
|
||||
}
|
||||
|
||||
return hash('sha256', $json, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the users rules.
|
||||
*
|
||||
@ -187,6 +203,7 @@ class ImportArrayStorage
|
||||
{
|
||||
/** @var JournalCollectorInterface $collector */
|
||||
$collector = app(JournalCollectorInterface::class);
|
||||
$collector->setUser($this->importJob->user);
|
||||
$collector->setAllAssetAccounts()
|
||||
->setTypes([TransactionType::TRANSFER])
|
||||
->withOpposingAccount();
|
||||
@ -198,20 +215,13 @@ class ImportArrayStorage
|
||||
/**
|
||||
* Check if the hash exists for the array the user wants to import.
|
||||
*
|
||||
* @param array $transaction
|
||||
* @param string $hash
|
||||
*
|
||||
* @return int|null
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function hashExists(array $transaction): ?int
|
||||
private function hashExists(string $hash): ?int
|
||||
{
|
||||
$json = json_encode($transaction);
|
||||
if ($json === false) {
|
||||
throw new FireflyException('Could not encode import array. Please see the logs.', $transaction); // @codeCoverageIgnore
|
||||
}
|
||||
$hash = hash('sha256', $json, false);
|
||||
|
||||
// find it!
|
||||
$entry = $this->journalRepos->findByHash($hash);
|
||||
if (null === $entry) {
|
||||
return null;
|
||||
@ -328,12 +338,13 @@ class ImportArrayStorage
|
||||
|
||||
foreach ($array as $index => $transaction) {
|
||||
Log::debug(sprintf('Now at item %d out of %d', $index + 1, $count));
|
||||
$existingId = $this->hashExists($transaction);
|
||||
$hash = $this->getHash($transaction);
|
||||
$existingId = $this->hashExists($hash);
|
||||
if (null !== $existingId) {
|
||||
$this->logDuplicateObject($transaction, $existingId);
|
||||
$this->repository->addErrorMessage(
|
||||
$this->importJob, sprintf(
|
||||
'Entry #%d ("%s") could not be imported. It already exists.',
|
||||
'Row #%d ("%s") could not be imported. It already exists.',
|
||||
$index, $transaction['description']
|
||||
)
|
||||
);
|
||||
@ -344,7 +355,7 @@ class ImportArrayStorage
|
||||
$this->logDuplicateTransfer($transaction);
|
||||
$this->repository->addErrorMessage(
|
||||
$this->importJob, sprintf(
|
||||
'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
|
||||
'Row #%d ("%s") could not be imported. Such a transfer already exists.',
|
||||
$index,
|
||||
$transaction['description']
|
||||
)
|
||||
@ -352,6 +363,7 @@ class ImportArrayStorage
|
||||
continue;
|
||||
}
|
||||
}
|
||||
$transaction['importHashV2'] = $hash;
|
||||
$toStore[] = $transaction;
|
||||
}
|
||||
$count = \count($toStore);
|
||||
|
||||
@ -475,6 +475,59 @@ class ImportJobRepository implements ImportJobRepositoryInterface
|
||||
$this->user = $user;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle upload for job.
|
||||
*
|
||||
* @param ImportJob $job
|
||||
* @param string $name
|
||||
* @param UploadedFile $file
|
||||
*
|
||||
* @return MessageBag
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag
|
||||
{
|
||||
$messages = new MessageBag;
|
||||
|
||||
if (!file_exists($fileName)) {
|
||||
$messages->add('notfound', sprintf('File not found: %s', $fileName));
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
$count = $job->attachments()->get()->filter(
|
||||
function (Attachment $att) use ($name) {
|
||||
return $att->filename === $name;
|
||||
}
|
||||
)->count();
|
||||
|
||||
if ($count > 0) {
|
||||
// don't upload, but also don't complain about it.
|
||||
Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name));
|
||||
|
||||
return new MessageBag;
|
||||
}
|
||||
$content = file_get_contents($fileName);
|
||||
$attachment = new Attachment; // create Attachment object.
|
||||
$attachment->user()->associate($job->user);
|
||||
$attachment->attachable()->associate($job);
|
||||
$attachment->md5 = md5($content);
|
||||
$attachment->filename = $name;
|
||||
$attachment->mime = 'plain/txt';
|
||||
$attachment->size = \strlen($content);
|
||||
$attachment->uploaded = 0;
|
||||
$attachment->save();
|
||||
$encrypted = Crypt::encrypt($content);
|
||||
|
||||
// store it:
|
||||
$this->uploadDisk->put($attachment->fileName(), $encrypted);
|
||||
$attachment->uploaded = 1; // update attachment
|
||||
$attachment->save();
|
||||
|
||||
// return it.
|
||||
return new MessageBag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle upload for job.
|
||||
*
|
||||
|
||||
@ -56,6 +56,17 @@ interface ImportJobRepositoryInterface
|
||||
*/
|
||||
public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag;
|
||||
|
||||
/**
|
||||
* Store file.
|
||||
*
|
||||
* @param ImportJob $job
|
||||
* @param string $name
|
||||
* @param string $fileName
|
||||
*
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag;
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
* @param array $transactions
|
||||
|
||||
@ -159,7 +159,7 @@ class JournalRepository implements JournalRepositoryInterface
|
||||
{
|
||||
return TransactionJournalMeta
|
||||
::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
|
||||
->where('data', $hash)
|
||||
->where('data', json_encode($hash))
|
||||
->where('name', 'importHashV2')
|
||||
->first(['journal_meta.*']);
|
||||
}
|
||||
|
||||
@ -168,7 +168,7 @@ class MappingConverger
|
||||
$newRole = 'opposing-id';
|
||||
break;
|
||||
}
|
||||
Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole));
|
||||
Log::debug(sprintf('Role was "%s", but because of mapping (mapped to #%d), role becomes "%s"', $role, $mapped, $newRole));
|
||||
|
||||
// also store the $mapped values in a "mappedValues" array.
|
||||
$this->mappedValues[$newRole][] = $mapped;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user