IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2024-12-23 15:40:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

First version of a web-based import status thing.

Browse Source
This commit is contained in:
James Cole 2016-08-13 21:51:01 +02:00
parent 2775690fc8
commit bbed5d0701
14 changed files with 383 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
app/Console/Commands/Import.php
View File

@ -13,6 +13,7 @@ namespace FireflyIII\Console\Commands;
use FireflyIII\Crud\Account\AccountCrud;
use FireflyIII\Import\Importer\ImporterInterface;
use FireflyIII\Import\ImportProcedure;
use FireflyIII\Import\ImportResult;
use FireflyIII\Import\ImportStorage;
use FireflyIII\Import\ImportValidator;
@ -71,47 +72,15 @@ class Import extends Command
return;
}
$job->status = 'import_running';
$job->save();
$this->line('Going to import job with key "' . $job->key . '" of type ' . $job->file_type);
$valid = array_keys(config('firefly.import_formats'));
$class = 'INVALID';
if (in_array($job->file_type, $valid)) {
$class = config('firefly.import_formats.' . $job->file_type);
}
/** @var ImporterInterface $importer */
$importer = app($class);
$importer->setJob($job);
// intercept logging by importer.
// intercept log entries and print them on the command line
$monolog = Log::getMonolog();
$handler = new CommandHandler($this);
$monolog->pushHandler($handler);
// create import entries
$collection = $importer->createImportEntries();
// validate / clean collection:
$validator = new ImportValidator($collection);
$validator->setUser($job->user);
if ($job->configuration['import-account'] != 0) {
$repository = app(AccountCrud::class, [$job->user]);
$validator->setDefaultImportAccount($repository->find($job->configuration['import-account']));
}
$cleaned = $validator->clean();
// then import collection:
$storage = new ImportStorage($cleaned);
$storage->setUser($job->user);
// and run store routine:
$result = $storage->store();
$job->status = 'import_complete';
$job->save();
$result = ImportProcedure::run($job);
/**
* @var int $index
@ -128,5 +97,6 @@ class Import extends Command
$this->line('The import has completed.');
}
}

63
app/Http/Controllers/ImportController.php
View File

@ -11,13 +11,14 @@ namespace FireflyIII\Http\Controllers;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Requests;
use FireflyIII\Http\Requests\ImportUploadRequest;
use FireflyIII\Import\ImportProcedure;
use FireflyIII\Import\Setup\SetupInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Http\Request;
use Log;
use Response;
use SplFileObject;
use Storage;
use Symfony\Component\HttpFoundation\File\UploadedFile;
@ -139,6 +140,35 @@ class ImportController extends Controller
return view('import.index', compact('subTitle', 'subTitleIcon', 'importFileTypes', 'defaultImportType'));
}
/**
* @param ImportJob $job
*
* @return \Illuminate\Http\JsonResponse
*/
public function json(ImportJob $job)
{
$result = [
'showPercentage' => false,
'status' => $job->status,
'key' => $job->key,
'started' => false,
'completed' => false,
'running' => false,
'percentage' => 0,
'steps' => $job->extended_status['total_steps'],
'stepsDone' => $job->extended_status['steps_done'],
'statusText' => trans('firefly.import_status_' . $job->status),
];
if ($job->status === 'import_running') {
$result['started'] = true;
$result['running'] = true;
$result['showPercentage'] = true;
$result['percentage'] = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
}
return Response::json($result);
}
/**
* Step 4. Save the configuration.
*
@ -237,6 +267,35 @@ class ImportController extends Controller
// depends of course on the data in the job.
}
/**
* @param ImportJob $job
*/
public function start(ImportJob $job)
{
if ($job->status == "settings_complete") {
ImportProcedure::run($job);
}
}
/**
* This is the last step before the import starts.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function status(ImportJob $job)
{
Log::debug('Now in status()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'status')) {
return $this->redirectToCorrectStep($job);
}
$subTitle = trans('firefy.import_status');
$subTitleIcon = 'fa-star';
return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));
}
/**
* This is step 2. It creates an Import Job. Stores the import.
*
@ -308,6 +367,8 @@ class ImportController extends Controller
return $job->status === 'import_configuration_saved';
case 'complete':
return $job->status === 'settings_complete';
case 'status':
return ($job->status === 'settings_complete') || ($job->status === 'import_running');
}
return false;

2
app/Http/routes.php
View File

@ -234,7 +234,7 @@ Route::group(
Route::get('/import/status/{importJob}', ['uses' => 'ImportController@status', 'as' => 'import.status']);
Route::get('/import/json/{importJob}', ['uses' => 'ImportController@json', 'as' => 'import.json']);
Route::get('/import/start/{importJob}', ['uses' => 'ImportController@run', 'as' => 'import.start']);
Route::post('/import/start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'import.start']);
/**

77
app/Import/ImportProcedure.php Normal file
View File

@ -0,0 +1,77 @@
<?php
/**
* ImportProcedure.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms
* of the MIT license. See the LICENSE file for details.
*/
declare(strict_types = 1);
namespace FireflyIII\Import;
use FireflyIII\Crud\Account\AccountCrud;
use FireflyIII\Import\Importer\ImporterInterface;
use FireflyIII\Models\ImportJob;
use Illuminate\Support\Collection;
/**
* Class ImportProcedure
*
* @package FireflyIII\Import
*/
class ImportProcedure
{
/**
* @param ImportJob $job
*
* @return Collection
*/
public static function run(ImportJob $job): Collection
{
// update job to say we started.
$job->status = 'import_running';
$job->save();
// create Importer
$valid = array_keys(config('firefly.import_formats'));
$class = 'INVALID';
if (in_array($job->file_type, $valid)) {
$class = config('firefly.import_formats.' . $job->file_type);
}
/** @var ImporterInterface $importer */
$importer = app($class);
$importer->setJob($job);
// create import entries
$collection = $importer->createImportEntries();
// validate / clean collection:
$validator = new ImportValidator($collection);
$validator->setUser($job->user);
$validator->setJob($job);
if ($job->configuration['import-account'] != 0) {
$repository = app(AccountCrud::class, [$job->user]);
$validator->setDefaultImportAccount($repository->find($job->configuration['import-account']));
}
$cleaned = $validator->clean();
// then import collection:
$storage = new ImportStorage($cleaned);
$storage->setJob($job);
$storage->setUser($job->user);
// and run store routine:
$result = $storage->store();
$job->status = 'import_complete';
$job->save();
return $result;
}
}

14
app/Import/ImportStorage.php
View File

@ -12,6 +12,7 @@ declare(strict_types = 1);
namespace FireflyIII\Import;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionJournal;
use FireflyIII\Models\TransactionType;
@ -29,7 +30,8 @@ class ImportStorage
/** @var Collection */
public $entries;
/** @var ImportJob */
public $job;
/** @var User */
public $user;
@ -44,6 +46,14 @@ class ImportStorage
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
}
/**
* @param User $user
*/
@ -62,6 +72,8 @@ class ImportStorage
foreach ($this->entries as $index => $entry) {
Log::debug(sprintf('--- import store start for row %d ---', $index));
$result = $this->storeSingle($index, $entry);
$this->job->addStepsDone(1);
sleep(1);
$collection->put($index, $result);
}
Log::notice(sprintf('Finished storing %d entry(ies).', $collection->count()));

16
app/Import/ImportValidator.php
View File

@ -15,6 +15,7 @@ use Carbon\Carbon;
use FireflyIII\Crud\Account\AccountCrudInterface;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use FireflyIII\User;
@ -35,6 +36,19 @@ class ImportValidator
/** @var User */
protected $user;
/** @var ImportJob */
public $job;
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
}
/**
* ImportValidator constructor.
*
@ -71,6 +85,8 @@ class ImportValidator
$entry = $this->setTransactionCurrency($entry);
$newCollection->put($index, $entry);
$this->job->addStepsDone(1);
sleep(1);
}
Log::notice(sprintf('Finished validating %d entry(ies).', $newCollection->count()));

3
app/Import/Importer/CsvImporter.php
View File

@ -64,6 +64,9 @@ class CsvImporter implements ImporterInterface
Log::debug(sprintf('Now going to import row %d.', $index));
$importEntry = $this->importSingleRow($index, $row);
$this->collection->put($line, $importEntry);
$this->job->addTotalSteps(3);
$this->job->addStepsDone(1);
sleep(1);
}
}
Log::debug(sprintf('Import collection contains %d entries', $this->collection->count()));

50
app/Models/ImportJob.php
View File

@ -28,7 +28,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
* @property string $key
* @property string $file_type
* @property string $status
* @property array $configuration
* @property array $configuration
* @property-read \FireflyIII\User $user
* @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereId($value)
* @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereCreatedAt($value)
@ -39,7 +39,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
* @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereStatus($value)
* @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereConfiguration($value)
* @mixin \Eloquent
* @property string $extended_status
* @property string $extended_status
* @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereExtendedStatus($value)
*/
class ImportJob extends Model
@ -62,6 +62,30 @@ class ImportJob extends Model
throw new NotFoundHttpException;
}
/**
* @param int $count
*/
public function addStepsDone(int $count)
{
$status = $this->extended_status;
$status['steps_done'] += $count;
$this->extended_status = $status;
$this->save();
}
/**
* @param int $count
*/
public function addTotalSteps(int $count)
{
$status = $this->extended_status;
$status['total_steps'] += $count;
$this->extended_status = $status;
$this->save();
}
/**
* @param $status
*/
@ -85,6 +109,20 @@ class ImportJob extends Model
return json_decode($value, true);
}
/**
* @param $value
*
* @return mixed
*/
public function getExtendedStatusAttribute($value)
{
if (strlen($value) == 0) {
return [];
}
return json_decode($value, true);
}
/**
* @param $value
*/
@ -93,6 +131,14 @@ class ImportJob extends Model
$this->attributes['configuration'] = json_encode($value);
}
/**
* @param $value
*/
public function setExtendedStatusAttribute($value)
{
$this->attributes['extended_status'] = json_encode($value);
}
/**
* @return string
*/

7
app/Repositories/ImportJob/ImportJobRepository.php
View File

@ -49,9 +49,10 @@ class ImportJobRepository implements ImportJobRepositoryInterface
if (is_null($existing->id)) {
$importJob = new ImportJob;
$importJob->user()->associate($this->user);
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'import_status_never_started';
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'import_status_never_started';
$importJob->extended_status = ['total_steps' => 0, 'steps_done' => 0,];
$importJob->save();
// breaks the loop:

15
app/Support/Migration/TestData.php
View File

@ -271,13 +271,14 @@ class TestData
$insert = [];
foreach ($this->data['import-jobs'] as $job) {
$insert[] = [
'created_at' => $this->time,
'updated_at' => $this->time,
'user_id' => $job['user_id'],
'file_type' => $job['file_type'],
'key' => $job['key'],
'status' => $job['status'],
'configuration' => json_encode($job['configuration']),
'created_at' => $this->time,
'updated_at' => $this->time,
'user_id' => $job['user_id'],
'file_type' => $job['file_type'],
'key' => $job['key'],
'status' => $job['status'],
'extended_status' => json_encode($job['extended_status']),
'configuration' => json_encode($job['configuration']),
];
}
DB::table('import_jobs')->insert($insert);

91
public/js/ff/import/status.js Normal file
View File

@ -0,0 +1,91 @@
/*
* status.js
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms
* of the MIT license. See the LICENSE file for details.
*/
/* globals $, jobImportUrl, jobStartUrl, token */
var startedImport = false;
var interval = 500;
$(function () {
"use strict";
// check status, every 500 ms.
setTimeout(checkImportStatus, 500);
});
function checkImportStatus() {
"use strict";
$.getJSON(jobImportUrl).success(reportOnJobImport).fail(failedJobImport);
}
function reportOnJobImport(data) {
"use strict";
console.log('Now in reportOnJobImport');
// update bar if it's a percentage or not:
var bar = $('#import-status-bar');
if (data.showPercentage) {
console.log('Has percentage.');
bar.addClass('progress-bar-success').removeClass('progress-bar-info');
bar.attr('aria-valuenow', data.percentage);
bar.css('width', data.percentage + '%');
$('#import-status-bar').text(data.stepsDone + '/' + data.steps);
if (data.percentage >= 100) {
console.log('Now import complete!');
bar.removeClass('active');
return;
}
} else {
$('#import-status-more-info').text('');
console.log('Has no percentage.');
bar.removeClass('progress-bar-success').addClass('progress-bar-info');
bar.attr('aria-valuenow', 100);
bar.css('width', '100%');
}
// update the message:
$('#import-status-txt').removeClass('text-danger').text(data.statusText);
// if the job has not actually started, do so now:
if (!data.started && !startedImport) {
console.log('Will now start job.');
$.post(jobStartUrl, {_token: token});
startedTheImport();
startedImport = true;
} else {
// trigger another check.
setTimeout(checkImportStatus, 500);
}
}
function startedTheImport() {
"use strict";
console.log('Started the import. Now starting over again.');
setTimeout(checkImportStatus, 500);
}
function failedJobImport(jqxhr, textStatus, error) {
"use strict";
// set status
$('#import-status-txt').addClass('text-danger').text(
"There was an error during the import routine. Please check the log files. The error seems to be: '" + textStatus + ' ' + error + "'."
);
// remove progress bar.
$('#import-status-holder').hide();
console.log('failedJobImport');
console.log(textStatus);
console.log(error);
}

5
resources/lang/en_US/firefly.php
View File

@ -767,6 +767,8 @@ return [
'configure_import' => 'Further configure your import',
'import_finish_configuration' => 'Finish configuration',
'settings_for_import' => 'Settings',
'import_status' => 'Import status',
'import_status_text' => 'The import is currently running, or will start momentarily.',
'import_complete' => 'Import configuration complete!',
'import_complete_text' => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.',
'import_download_config' => 'Download configuration',
@ -777,4 +779,7 @@ return [
'import' => 'Import',
'import_intro_text' => 'Welcome to the Firefly III data import routine. At the moment, this routine can help you import files into Firefly. To do so, you must download or export transactions from other systems or software, and upload them here. The next steps will let you help Firefly III determin what the content is of your file, and how to handle it. Please select a file, and read all instructions carefully.',
'import_file_help' => 'Select your file',
'import_status_settings_complete' => 'The import is ready to start.',
'import_status_import_complete' => 'The import has completed.',
'import_status_import_running' => 'The import is currently running. Please be patient. An apparent lack of progress may be a trick of the light.',
];

47
resources/views/import/status.twig Normal file
View File

@ -0,0 +1,47 @@
{% extends "./layout/default.twig" %}
{% block breadcrumbs %}
{{ Breadcrumbs.renderIfExists }}
{% endblock %}
{% block content %}
<div class="row">
<div class="col-lg-12 col-md-12 col-sm-12">
<div class="box box-default">
<div class="box-header with-border">
<h3 class="box-title">{{ 'import_status'|_ }}</h3>
</div>
<div class="box-body">
<p>
{{ 'import_status_text'|_ }}
</p>
<div class="row">
<div class="col-lg-6 col-lg-offset-3">
<div class="progress" id="import-status-holder">
<div id="import-status-bar" class="progress-bar progress-bar-info active progress-bar-striped" role="progressbar"
aria-valuenow="100" aria-valuemin="0"
aria-valuemax="100" style="width: 100%">
</div>
</div>
<p id="import-status-txt"></p>
<p id="import-status-more-info"></p>
</div>
</div>
</div>
</div>
</div>
</div>
{% endblock %}
{% block scripts %}
<script type="text/javascript">
var jobKey = '{{ job.key }}';
var jobImportUrl = '{{ route('import.json', [job.key]) }}';
var jobStartUrl = '{{ route('import.start', [job.key]) }}';
var token = '{{ csrf_token() }}';
</script>
<script type="text/javascript" src="js/ff/import/status.js"></script>
{% endblock %}
{% block styles %}
{% endblock %}

4
storage/database/seed.import-test.json
View File

@ -96,6 +96,10 @@
"key": "testImport",
"file_type": "csv",
"status": "settings_complete",
"extended_status": {
"steps_done": 0,
"total_steps": 0
},
"configuration": {
"has-headers": false,
"date-format": "Ymd",