IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

New code for updated import routine.

Browse Source
This commit is contained in:
James Cole 2018-05-01 20:47:38 +02:00
parent cd75224cdd
commit ccda71ff8e
16 changed files with 262 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
app/Http/Controllers/Import/IndexController.php
View File

@ -58,7 +58,7 @@ class IndexController extends Controller
}
/**
* Creates a new import job for $importProvider with the default (global) job configuration.
* Creates a new import job for $importProvider.
*
* @param string $importProvider
*
@ -73,7 +73,6 @@ class IndexController extends Controller
// if job provider has no prerequisites:
if (!(bool)config(sprintf('import.has_prereq.%s', $importProvider))) {
// if job provider also has no configuration:
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
$this->repository->updateStatus($importJob, 'ready_to_run');
@ -93,11 +92,11 @@ class IndexController extends Controller
if (!class_exists($class)) {
throw new FireflyException(sprintf('No class to handle configuration for "%s".', $importProvider)); // @codeCoverageIgnore
}
/** @var PrerequisitesInterface $object */
$object = app($class);
$object->setUser(auth()->user());
/** @var PrerequisitesInterface $providerPre */
$providerPre = app($class);
$providerPre->setUser(auth()->user());
if (!$object->isComplete()) {
if (!$providerPre->isComplete()) {
// redirect to global prerequisites
return redirect(route('import.prerequisites.index', [$importProvider, $importJob->key]));
}
@ -110,42 +109,6 @@ class IndexController extends Controller
}
// /**
// * Generate a JSON file of the job's configuration and send it to the user.
// *
// * @param ImportJob $job
// *
// * @return LaravelResponse
// */
// public function download(ImportJob $job)
// {
// Log::debug('Now in download()', ['job' => $job->key]);
// $config = $job->configuration;
//
// // This is CSV import specific:
// $config['column-roles-complete'] = false;
// $config['column-mapping-complete'] = false;
// $config['initial-config-complete'] = false;
// $config['has-file-upload'] = false;
// $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter'];
// unset($config['stage']);
//
// $result = json_encode($config, JSON_PRETTY_PRINT);
// $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\'));
//
// /** @var LaravelResponse $response */
// $response = response($result, 200);
// $response->header('Content-disposition', 'attachment; filename=' . $name)
// ->header('Content-Type', 'application/json')
// ->header('Content-Description', 'File Transfer')
// ->header('Connection', 'Keep-Alive')
// ->header('Expires', '0')
// ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0')
// ->header('Pragma', 'public')
// ->header('Content-Length', \strlen($result));
//
// return $response;
// }
/**
* General import index.
@ -252,4 +215,42 @@ class IndexController extends Controller
//
// throw new FireflyException('Job did not complete successfully. Please review the log files.');
// }
// /**
// * Generate a JSON file of the job's configuration and send it to the user.
// *
// * @param ImportJob $job
// *
// * @return LaravelResponse
// */
// public function download(ImportJob $job)
// {
// Log::debug('Now in download()', ['job' => $job->key]);
// $config = $job->configuration;
//
// // This is CSV import specific:
// $config['column-roles-complete'] = false;
// $config['column-mapping-complete'] = false;
// $config['initial-config-complete'] = false;
// $config['has-file-upload'] = false;
// $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter'];
// unset($config['stage']);
//
// $result = json_encode($config, JSON_PRETTY_PRINT);
// $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\'));
//
// /** @var LaravelResponse $response */
// $response = response($result, 200);
// $response->header('Content-disposition', 'attachment; filename=' . $name)
// ->header('Content-Type', 'application/json')
// ->header('Content-Description', 'File Transfer')
// ->header('Connection', 'Keep-Alive')
// ->header('Expires', '0')
// ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0')
// ->header('Pragma', 'public')
// ->header('Content-Length', \strlen($result));
//
// return $response;
// }
}

69
app/Http/Controllers/Import/JobConfigurationController.php
View File

@ -61,63 +61,80 @@ class JobConfigurationController extends Controller
/**
* Configure the job. This method is returned to until job is deemed "configured".
*
* @param ImportJob $job
* @param ImportJob $importJob
*
* @return \Illuminate\Contracts\View\Factory|\Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|\Illuminate\View\View
*
* @throws FireflyException
*/
public function index(ImportJob $job)
public function index(ImportJob $importJob)
{
// if provider has no config, just push it through
$importProvider = $job->provider;
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
$this->repository->updateStatus($job, 'ready_to_run');
return redirect(route('import.job.status.index', [$job->key]));
// catch impossible status:
$allowed = ['has_prereq', 'need_job_config', 'has_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
session()->flash('error', trans('import.bad_job_status'));
return redirect(route('import.index'));
}
Log::debug(sprintf('Now in JobConfigurationController::index() with job "%s" and status "%s"', $importJob->key, $importJob->status));
// if provider has no config, just push it through:
$importProvider = $importJob->provider;
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
Log::debug('Job needs no config, is ready to run!');
$this->repository->updateStatus($importJob ,'ready_to_run');
return redirect(route('import.job.status.index', [$importProvider->key]));
}
// create configuration class:
$configurator = $this->makeConfigurator($job);
$configurator = $this->makeConfigurator($importJob);
// is the job already configured?
if ($configurator->configurationComplete()) {
$this->repository->updateStatus($job, 'ready_to_run');
Log::debug('Config is complete, set status to ready_to_run.');
$this->repository->updateStatus($importJob, 'ready_to_run');
return redirect(route('import.job.status.index', [$job->key]));
return redirect(route('import.job.status.index', [$importJob->key]));
}
$this->repository->updateStatus($job, 'configuring');
$view = $configurator->getNextView();
$data = $configurator->getNextData();
$subTitle = trans('firefly.import_config_bread_crumb');
$subTitleIcon = 'fa-wrench';
return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
return view($view, compact('data', 'importJob', 'subTitle', 'subTitleIcon'));
}
/**
* Store the configuration. Returns to "configure" method until job is configured.
*
* @param Request $request
* @param ImportJob $job
* @param ImportJob $importJob
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*
* @throws FireflyException
*/
public function post(Request $request, ImportJob $job)
public function post(Request $request, ImportJob $importJob)
{
Log::debug('Now in postConfigure()', ['job' => $job->key]);
$configurator = $this->makeConfigurator($job);
// catch impossible status:
$allowed = ['has_prereq', 'need_job_config', 'has_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
session()->flash('error', trans('import.bad_job_status'));
return redirect(route('import.index'));
}
Log::debug('Now in postConfigure()', ['job' => $importJob->key]);
$configurator = $this->makeConfigurator($importJob);
// is the job already configured?
if ($configurator->configurationComplete()) {
$this->repository->updateStatus($job, 'ready_to_run');
$this->repository->updateStatus($importJob, 'ready_to_run');
return redirect(route('import.job.status.index', [$job->key]));
return redirect(route('import.job.status.index', [$importJob->key]));
}
$data = $request->all();
@ -128,27 +145,27 @@ class JobConfigurationController extends Controller
}
// return to configure
return redirect(route('import.job.configuration.index', [$job->key]));
return redirect(route('import.job.configuration.index', [$importJob->key]));
}
/**
* @param ImportJob $job
* @param ImportJob $importJob
*
* @return JobConfiguratorInterface
*
* @throws FireflyException
*/
private function makeConfigurator(ImportJob $job): JobConfiguratorInterface
private function makeConfigurator(ImportJob $importJob): JobConfiguratorInterface
{
$key = sprintf('import.configuration.%s', $job->provider);
$key = sprintf('import.configuration.%s', $importJob->provider);
$className = (string)config($key);
if (null === $className || !class_exists($className)) {
throw new FireflyException(sprintf('Cannot find configurator class for job with provider "%s".', $job->provider)); // @codeCoverageIgnore
throw new FireflyException(sprintf('Cannot find configurator class for job with provider "%s".', $importJob->provider)); // @codeCoverageIgnore
}
Log::debug(sprintf('Going to create class "%s"', $className));
/** @var JobConfiguratorInterface $configurator */
$configurator = app($className);
$configurator->setJob($job);
$configurator->setJob($importJob);
return $configurator;
}

33
app/Http/Controllers/Import/JobStatusController.php
View File

@ -86,10 +86,10 @@ class JobStatusController extends Controller
*
* @return JsonResponse
*/
public function json(ImportJob $job): JsonResponse
public function json(ImportJob $importJob): JsonResponse
{
$json = [
'status' => $job->status,
'status' => $importJob->status,
];
return response()->json($json);
@ -101,20 +101,34 @@ class JobStatusController extends Controller
* @return JsonResponse
* @throws FireflyException
*/
public function start(ImportJob $job): JsonResponse
public function start(ImportJob $importJob): JsonResponse
{
$importProvider = $job->provider;
// catch impossible status:
$allowed = ['ready_to_run'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not ready.');
session()->flash('error', trans('import.bad_job_status'));
return redirect(route('import.index'));
}
$importProvider = $importJob->provider;
$key = sprintf('import.routine.%s', $importProvider);
$className = config($key);
if (null === $className || !class_exists($className)) {
return response()->json(['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]);
}
// if the job is set to "provider_finished", we should be able to store transactions
// generated by the provider.
// otherwise, just continue.
// set job to be running:
$this->repository->setStatus($job, 'running');
$this->repository->setStatus($importJob, 'running');
/** @var RoutineInterface $routine */
$routine = app($className);
$routine->setJob($job);
$routine->setJob($importJob);
try {
$routine->run();
} catch (FireflyException $e) {
@ -123,16 +137,13 @@ class JobStatusController extends Controller
Log::error($e->getTraceAsString());
// set job errored out:
$this->repository->setStatus($job, 'errored');
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => $message]);
}
// set job finished this step:
$this->repository->setStatus($job, 'stage_finished');
// expect nothing from routine, just return OK to user.
return response()->json(['status' => 'OK', 'message' => 'finished']);
return response()->json(['status' => 'OK', 'message' => 'stage_finished']);
}
// /**

29
app/Http/Controllers/Import/PrerequisitesController.php
View File

@ -63,9 +63,8 @@ class PrerequisitesController extends Controller
}
/**
* Once there are no prerequisites, this method will create an importjob object and
* redirect the user to a view where this object can be used by a bank specific
* class to process.
* This method will process and store import provider global prerequisites
* such as API keys.
*
* @param string $importProvider
* @param ImportJob $importJob
@ -75,6 +74,14 @@ class PrerequisitesController extends Controller
*/
public function index(string $importProvider, ImportJob $importJob = null)
{
// catch impossible status:
$allowed = ['new'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
session()->flash('error', trans('import.bad_job_status'));
return redirect(route('import.index'));
}
app('view')->share('subTitle', trans('import.prerequisites_breadcrumb_' . $importProvider));
$class = (string)config(sprintf('import.prerequisites.%s', $importProvider));
if (!class_exists($class)) {
@ -84,9 +91,12 @@ class PrerequisitesController extends Controller
$object = app($class);
$object->setUser(auth()->user());
// TODO if prerequisites have been met and job is not null, just skip this step.
if (null !== $importJob && $object->isComplete()) {
// set job to
// update job:
$this->repository->setStatus($importJob, 'has_prereq');
// redirect to job config:
return redirect(route('import.job.configuration.index', [$importJob->key]));
}
@ -117,6 +127,15 @@ class PrerequisitesController extends Controller
{
Log::debug(sprintf('Now in postPrerequisites for %s', $importProvider));
// catch impossible status:
$allowed = ['new'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
session()->flash('error', trans('import.bad_job_status'));
return redirect(route('import.index'));
}
$class = (string)config(sprintf('import.prerequisites.%s', $importProvider));
if (!class_exists($class)) {
throw new FireflyException(sprintf('Cannot find class %s', $class)); // @codeCoverageIgnore

31
app/Import/Routine/FakeRoutine.php
View File

@ -26,8 +26,10 @@ namespace FireflyIII\Import\Routine;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Routine\Fake\StageAhoyHandler;
use FireflyIII\Support\Import\Routine\Fake\StageFinalHandler;
use FireflyIII\Support\Import\Routine\Fake\StageNewHandler;
use Log;
/**
* Class FakeRoutine
@ -51,9 +53,9 @@ class FakeRoutine implements RoutineInterface
/**
* Fake import routine has three stages:
*
* "new": will quietly log gibberish for 15 seconds, then switch to stage "ahoy"
* unless "ahoy" has been done already. If so, jump to stage "final".
* "ahoy": will log some nonsense and then drop job into "need_extra_config" to force it back to the job config routine.
* "new": will quietly log gibberish for 15 seconds, then switch to stage "ahoy".
* will also set status to "ready_to_run" so it will arrive here again.
* "ahoy": will log some nonsense and then drop job into status:"need_job_config" to force it back to the job config routine.
* "final": will do some logging, sleep for 10 seconds and then finish. Generates 5 random transactions.
*
* @return bool
@ -61,12 +63,33 @@ class FakeRoutine implements RoutineInterface
*/
public function run(): void
{
Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->job->status));
if ($this->job->status !== 'running') {
throw new FireflyException('This fake job should not be started.');
}
switch ($this->job->stage) {
default:
throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->job->stage));
case 'new':
$handler = new StageNewHandler;
$handler->run();
$this->repository->setStage($this->job, 'ahoy');
// set job finished this step:
$this->repository->setStatus($this->job, 'ready_to_run');
return;
case 'ahoy':
$handler = new StageAhoyHandler;
$handler->run();
$this->repository->setStatus($this->job, 'need_job_config');
$this->repository->setStage($this->job, 'final');
break;
case 'final':
$handler = new StageFinalHandler;
$transactions = $handler->getTransactions();
$this->repository->setStatus($this->job, 'provider_finished');
$this->repository->setStage($this->job, 'final');
}
}

4
app/Import/Routine/RoutineInterface.php
View File

@ -31,6 +31,10 @@ use FireflyIII\Models\ImportJob;
interface RoutineInterface
{
/**
* At the end of each run(), the import routine must set the job to the expected status.
*
* The final status of the routine must be "provider_finished".
*
* @return bool
* @throws FireflyException
*/

1
app/Repositories/ImportJob/ImportJobRepository.php
View File

@ -356,6 +356,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
*/
public function setStatus(ImportJob $job, string $status): ImportJob
{
Log::debug(sprintf('Set status of job "%s" to "%s"', $job->key, $status));
$job->status = $status;
$job->save();

45
app/Support/Import/Routine/Fake/StageAhoyHandler.php Normal file
View File

@ -0,0 +1,45 @@
<?php
/**
* StageAhoyHandler.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\Fake;
use FireflyIII\Exceptions\FireflyException;
use Log;
/**
* Class StageAhoyHandler
*/
class StageAhoyHandler
{
/**
* @throws FireflyException
*/
public function run(): void
{
for ($i = 0; $i < 15; $i++) {
Log::debug(sprintf('Am now in stage AHOY hander, sleeping... (%d)', $i));
sleep(1);
}
}
}

31
app/Support/Import/Routine/Fake/StageFinalHandler.php Normal file
View File

@ -0,0 +1,31 @@
<?php
namespace FireflyIII\Support\Import\Routine\Fake;
/**
* Class StageFinalHandler
*
* @package FireflyIII\Support\Import\Routine\Fake
*/
class StageFinalHandler
{
/**
* @return array
*/
public function getTransactions(): array
{
$transactions = [];
for ($i = 0; $i < 5; $i++) {
$transaction = [];
$transactions[] = $transaction;
}
return $transactions;
}
}

3
app/Support/Import/Routine/Fake/StageNewHandler.php
View File

@ -23,6 +23,7 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\Fake;
use FireflyIII\Exceptions\FireflyException;
use Log;
/**
@ -31,7 +32,7 @@ use Log;
class StageNewHandler
{
/**
*
* @throws FireflyException
*/
public function run(): void
{

4
config/import.php
View File

@ -38,7 +38,7 @@ use FireflyIII\Import\Routine\SpectreRoutine;
return [
'enabled' => [
'fake' => false,
'fake' => true,
'file' => true,
'bunq' => true,
'spectre' => true,
@ -65,7 +65,7 @@ return [
'yodlee' => false,
],
'has_config' => [
'fake' => false,
'fake' => true,
'file' => true,
'bunq' => true,
'spectre' => true,

13
public/js/ff/import/status_v2.js
View File

@ -24,8 +24,9 @@ var timeOutId;
var hasStartedJob = false;
var checkInitialInterval = 1000;
var checkNextInterval = 500;
var maxLoops = 20;
var maxLoops = 60;
var totalLoops = 0;
var startCount = 0;
$(function () {
"use strict";
@ -50,6 +51,10 @@ function reportOnJobStatus(data) {
console.log(data);
switch (data.status) {
case "ready_to_run":
if (startCount > 0) {
hasStartedJob = false;
}
startCount++;
startJob();
checkOnJob();
break;
@ -57,8 +62,14 @@ function reportOnJobStatus(data) {
showProgressBox();
checkOnJob();
break;
case "need_job_config":
// redirect user to configuration for this job.
window.location.replace(jobConfigurationUri);
break;
default:
console.error('Cannot handle status ' + data.status);
}
}

3
resources/lang/en_US/import.php
View File

@ -180,6 +180,7 @@ return [
// index of import:
'general_index_title' => 'Import a file',
'general_index_intro' => 'Welcome to Firefly III\'s import routine. There are a few ways of importing data into Firefly III, displayed here as buttons.',
'bad_job_status' => 'You cannot access this page when the job is at this point. Sorry!',
// import provider strings (index):
'button_fake' => 'Fake an import',
@ -217,7 +218,7 @@ return [
'do_config_quovo' => 'Configuration for imports from Quovo',
// job configuration:
'job_configuration_breadcrumb' => 'Configuration for job ":key"',
'job_configuration_breadcrumb' => 'Configuration for job ":key"',
// import index page:
'index_breadcrumb' => 'Index',

2
resources/views/import/fake/enter-artist.twig
View File

@ -20,7 +20,7 @@
</div>
</div>
<form method="POST" action="{{ route('import.job.configuration.post', job.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
<form method="POST" action="{{ route('import.job.configuration.post', importJob.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
<input name="_token" type="hidden" value="{{ csrf_token() }}">
<div class="row">
<div class="col-lg-12">

2
resources/views/import/fake/enter-song.twig
View File

@ -20,7 +20,7 @@
</div>
</div>
<form method="POST" action="{{ route('import.job.configuration.post', job.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
<form method="POST" action="{{ route('import.job.configuration.post', importJob.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
<input name="_token" type="hidden" value="{{ csrf_token() }}">
<div class="row">
<div class="col-lg-12">

4
resources/views/import/status.twig
View File

@ -170,7 +170,7 @@
var jobStatusUri = '{{ route('import.job.status.json', [importJob.key]) }}';
var jobStartUri = '{{ route('import.job.start', [importJob.key]) }}';
var jobConfigurationUri = '{{ route('import.job.configuration.index', [importJob.key]) }}';
// some useful translations.
{#var langImportTimeOutError = '(time out thing)';#}
@ -178,7 +178,7 @@
{#var langImportMultiError = '{{ trans('import.status_errors_multi')|escape('js') }}';#}
{#var jobConfigureUri = '#}{#{{ route('import.configure', [job.key]) }}#}{#';#}
var token = '{{ csrf_token() }}';
var job = {{ job|json_encode|raw }};
</script>