First version of a web-based import status thing.

app/Console/Commands/Import.php

@@ -13,6 +13,7 @@ namespace FireflyIII\Console\Commands;
 
 use FireflyIII\Crud\Account\AccountCrud;
 use FireflyIII\Import\Importer\ImporterInterface;
+use FireflyIII\Import\ImportProcedure;
 use FireflyIII\Import\ImportResult;
 use FireflyIII\Import\ImportStorage;
 use FireflyIII\Import\ImportValidator;
@@ -71,47 +72,15 @@ class Import extends Command
 
             return;
         }
-        $job->status = 'import_running';
-        $job->save();
 
         $this->line('Going to import job with key "' . $job->key . '" of type ' . $job->file_type);
-        $valid = array_keys(config('firefly.import_formats'));
-        $class = 'INVALID';
-        if (in_array($job->file_type, $valid)) {
-            $class = config('firefly.import_formats.' . $job->file_type);
-        }
 
-        /** @var ImporterInterface $importer */
+        // intercept log entries and print them on the command line
-        $importer = app($class);
-        $importer->setJob($job);
-        // intercept logging by importer.
         $monolog = Log::getMonolog();
         $handler = new CommandHandler($this);
-
         $monolog->pushHandler($handler);
 
-        // create import entries
+        $result = ImportProcedure::run($job);
-        $collection = $importer->createImportEntries();
-
-        // validate / clean collection:
-        $validator = new ImportValidator($collection);
-        $validator->setUser($job->user);
-        if ($job->configuration['import-account'] != 0) {
-            $repository = app(AccountCrud::class, [$job->user]);
-            $validator->setDefaultImportAccount($repository->find($job->configuration['import-account']));
-        }
-
-        $cleaned = $validator->clean();
-
-        // then import collection:
-        $storage = new ImportStorage($cleaned);
-        $storage->setUser($job->user);
-
-        // and run store routine:
-        $result = $storage->store();
-
-        $job->status = 'import_complete';
-        $job->save();
 
         /**
          * @var int          $index
@@ -128,5 +97,6 @@ class Import extends Command
 
 
         $this->line('The import has completed.');
+
     }
 }

app/Http/Controllers/ImportController.php

@@ -11,13 +11,14 @@ namespace FireflyIII\Http\Controllers;
 
 use Crypt;
 use FireflyIII\Exceptions\FireflyException;
-use FireflyIII\Http\Requests;
 use FireflyIII\Http\Requests\ImportUploadRequest;
+use FireflyIII\Import\ImportProcedure;
 use FireflyIII\Import\Setup\SetupInterface;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
 use Illuminate\Http\Request;
 use Log;
+use Response;
 use SplFileObject;
 use Storage;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
@@ -139,6 +140,35 @@ class ImportController extends Controller
         return view('import.index', compact('subTitle', 'subTitleIcon', 'importFileTypes', 'defaultImportType'));
     }
 
+    /**
+     * @param ImportJob $job
+     *
+     * @return \Illuminate\Http\JsonResponse
+     */
+    public function json(ImportJob $job)
+    {
+        $result = [
+            'showPercentage' => false,
+            'status'         => $job->status,
+            'key'            => $job->key,
+            'started'        => false,
+            'completed'      => false,
+            'running'        => false,
+            'percentage'     => 0,
+            'steps'          => $job->extended_status['total_steps'],
+            'stepsDone'      => $job->extended_status['steps_done'],
+            'statusText'     => trans('firefly.import_status_' . $job->status),
+        ];
+        if ($job->status === 'import_running') {
+            $result['started']        = true;
+            $result['running']        = true;
+            $result['showPercentage'] = true;
+            $result['percentage']     = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
+        }
+
+        return Response::json($result);
+    }
+
     /**
      * Step 4. Save the configuration.
      *
@@ -237,6 +267,35 @@ class ImportController extends Controller
         // depends of course on the data in the job.
     }
 
+    /**
+     * @param ImportJob $job
+     */
+    public function start(ImportJob $job)
+    {
+        if ($job->status == "settings_complete") {
+            ImportProcedure::run($job);
+        }
+    }
+
+    /**
+     * This is the last step before the import starts.
+     *
+     * @param ImportJob $job
+     *
+     * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
+     */
+    public function status(ImportJob $job)
+    {
+        Log::debug('Now in status()', ['job' => $job->key]);
+        if (!$this->jobInCorrectStep($job, 'status')) {
+            return $this->redirectToCorrectStep($job);
+        }
+        $subTitle     = trans('firefy.import_status');
+        $subTitleIcon = 'fa-star';
+
+        return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));
+    }
+
     /**
      * This is step 2. It creates an Import Job. Stores the import.
      *
@@ -308,6 +367,8 @@ class ImportController extends Controller
                 return $job->status === 'import_configuration_saved';
             case 'complete':
                 return $job->status === 'settings_complete';
+            case 'status':
+                return ($job->status === 'settings_complete') || ($job->status === 'import_running');
         }
 
         return false;

app/Http/routes.php

@@ -234,7 +234,7 @@ Route::group(
 
     Route::get('/import/status/{importJob}', ['uses' => 'ImportController@status', 'as' => 'import.status']);
     Route::get('/import/json/{importJob}', ['uses' => 'ImportController@json', 'as' => 'import.json']);
-    Route::get('/import/start/{importJob}', ['uses' => 'ImportController@run', 'as' => 'import.start']);
+    Route::post('/import/start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'import.start']);
 
 
     /**

app/Import/ImportProcedure.php

@@ -0,0 +1,77 @@
+<?php
+/**
+ * ImportProcedure.php
+ * Copyright (C) 2016 thegrumpydictator@gmail.com
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE file for details.
+ */
+
+declare(strict_types = 1);
+
+namespace FireflyIII\Import;
+
+use FireflyIII\Crud\Account\AccountCrud;
+use FireflyIII\Import\Importer\ImporterInterface;
+use FireflyIII\Models\ImportJob;
+use Illuminate\Support\Collection;
+
+/**
+ * Class ImportProcedure
+ *
+ * @package FireflyIII\Import
+ */
+class ImportProcedure
+{
+
+    /**
+     * @param ImportJob $job
+     *
+     * @return Collection
+     */
+    public static function run(ImportJob $job): Collection
+    {
+        // update job to say we started.
+        $job->status = 'import_running';
+        $job->save();
+
+        // create Importer
+        $valid = array_keys(config('firefly.import_formats'));
+        $class = 'INVALID';
+        if (in_array($job->file_type, $valid)) {
+            $class = config('firefly.import_formats.' . $job->file_type);
+        }
+
+        /** @var ImporterInterface $importer */
+        $importer = app($class);
+        $importer->setJob($job);
+
+        // create import entries
+        $collection = $importer->createImportEntries();
+
+        // validate / clean collection:
+        $validator = new ImportValidator($collection);
+        $validator->setUser($job->user);
+        $validator->setJob($job);
+        if ($job->configuration['import-account'] != 0) {
+            $repository = app(AccountCrud::class, [$job->user]);
+            $validator->setDefaultImportAccount($repository->find($job->configuration['import-account']));
+        }
+
+        $cleaned = $validator->clean();
+
+        // then import collection:
+        $storage = new ImportStorage($cleaned);
+        $storage->setJob($job);
+        $storage->setUser($job->user);
+
+        // and run store routine:
+        $result = $storage->store();
+
+        $job->status = 'import_complete';
+        $job->save();
+
+        return $result;
+    }
+
+}

app/Import/ImportStorage.php

@@ -12,6 +12,7 @@ declare(strict_types = 1);
 namespace FireflyIII\Import;
 
 use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Models\ImportJob;
 use FireflyIII\Models\Transaction;
 use FireflyIII\Models\TransactionJournal;
 use FireflyIII\Models\TransactionType;
@@ -29,7 +30,8 @@ class ImportStorage
 
     /** @var  Collection */
     public $entries;
-
+    /** @var  ImportJob */
+    public $job;
     /** @var  User */
     public $user;
 
@@ -44,6 +46,14 @@ class ImportStorage
 
     }
 
+    /**
+     * @param ImportJob $job
+     */
+    public function setJob(ImportJob $job)
+    {
+        $this->job = $job;
+    }
+
     /**
      * @param User $user
      */
@@ -62,6 +72,8 @@ class ImportStorage
         foreach ($this->entries as $index => $entry) {
             Log::debug(sprintf('--- import store start for row %d ---', $index));
             $result = $this->storeSingle($index, $entry);
+            $this->job->addStepsDone(1);
+            sleep(1);
             $collection->put($index, $result);
         }
         Log::notice(sprintf('Finished storing %d entry(ies).', $collection->count()));

app/Import/ImportValidator.php

@@ -15,6 +15,7 @@ use Carbon\Carbon;
 use FireflyIII\Crud\Account\AccountCrudInterface;
 use FireflyIII\Models\Account;
 use FireflyIII\Models\AccountType;
+use FireflyIII\Models\ImportJob;
 use FireflyIII\Models\TransactionType;
 use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
 use FireflyIII\User;
@@ -35,6 +36,19 @@ class ImportValidator
     /** @var  User */
     protected $user;
 
+    /** @var  ImportJob */
+    public $job;
+
+    /**
+     * @param ImportJob $job
+     */
+    public function setJob(ImportJob $job)
+    {
+        $this->job = $job;
+    }
+
+
+
     /**
      * ImportValidator constructor.
      *
@@ -71,6 +85,8 @@ class ImportValidator
             $entry = $this->setTransactionCurrency($entry);
 
             $newCollection->put($index, $entry);
+            $this->job->addStepsDone(1);
+            sleep(1);
         }
         Log::notice(sprintf('Finished validating %d entry(ies).', $newCollection->count()));
 

app/Import/Importer/CsvImporter.php

@@ -64,6 +64,9 @@ class CsvImporter implements ImporterInterface
                 Log::debug(sprintf('Now going to import row %d.', $index));
                 $importEntry = $this->importSingleRow($index, $row);
                 $this->collection->put($line, $importEntry);
+                $this->job->addTotalSteps(3);
+                $this->job->addStepsDone(1);
+                sleep(1);
             }
         }
         Log::debug(sprintf('Import collection contains %d entries', $this->collection->count()));

app/Models/ImportJob.php

@@ -28,7 +28,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
  * @property string                $key
  * @property string                $file_type
  * @property string                $status
- * @property array                $configuration
+ * @property array                 $configuration
  * @property-read \FireflyIII\User $user
  * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereId($value)
  * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereCreatedAt($value)
@@ -39,7 +39,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
  * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereStatus($value)
  * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereConfiguration($value)
  * @mixin \Eloquent
- * @property string $extended_status
+ * @property string                $extended_status
  * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereExtendedStatus($value)
  */
 class ImportJob extends Model
@@ -62,6 +62,30 @@ class ImportJob extends Model
         throw new NotFoundHttpException;
     }
 
+    /**
+     * @param int $count
+     */
+    public function addStepsDone(int $count)
+    {
+        $status = $this->extended_status;
+        $status['steps_done'] += $count;
+        $this->extended_status = $status;
+        $this->save();
+
+    }
+
+    /**
+     * @param int $count
+     */
+    public function addTotalSteps(int $count)
+    {
+        $status = $this->extended_status;
+        $status['total_steps'] += $count;
+        $this->extended_status = $status;
+        $this->save();
+
+    }
+
     /**
      * @param $status
      */
@@ -85,6 +109,20 @@ class ImportJob extends Model
         return json_decode($value, true);
     }
 
+    /**
+     * @param $value
+     *
+     * @return mixed
+     */
+    public function getExtendedStatusAttribute($value)
+    {
+        if (strlen($value) == 0) {
+            return [];
+        }
+
+        return json_decode($value, true);
+    }
+
     /**
      * @param $value
      */
@@ -93,6 +131,14 @@ class ImportJob extends Model
         $this->attributes['configuration'] = json_encode($value);
     }
 
+    /**
+     * @param $value
+     */
+    public function setExtendedStatusAttribute($value)
+    {
+        $this->attributes['extended_status'] = json_encode($value);
+    }
+
     /**
      * @return string
      */

app/Repositories/ImportJob/ImportJobRepository.php

@@ -49,9 +49,10 @@ class ImportJobRepository implements ImportJobRepositoryInterface
             if (is_null($existing->id)) {
                 $importJob = new ImportJob;
                 $importJob->user()->associate($this->user);
-                $importJob->file_type = $fileType;
+                $importJob->file_type       = $fileType;
-                $importJob->key    = Str::random(12);
+                $importJob->key             = Str::random(12);
-                $importJob->status = 'import_status_never_started';
+                $importJob->status          = 'import_status_never_started';
+                $importJob->extended_status = ['total_steps' => 0, 'steps_done' => 0,];
                 $importJob->save();
 
                 // breaks the loop:

app/Support/Migration/TestData.php

@@ -271,13 +271,14 @@ class TestData
         $insert = [];
         foreach ($this->data['import-jobs'] as $job) {
             $insert[] = [
-                'created_at'    => $this->time,
+                'created_at'      => $this->time,
-                'updated_at'    => $this->time,
+                'updated_at'      => $this->time,
-                'user_id'       => $job['user_id'],
+                'user_id'         => $job['user_id'],
-                'file_type'     => $job['file_type'],
+                'file_type'       => $job['file_type'],
-                'key'           => $job['key'],
+                'key'             => $job['key'],
-                'status'        => $job['status'],
+                'status'          => $job['status'],
-                'configuration' => json_encode($job['configuration']),
+                'extended_status' => json_encode($job['extended_status']),
+                'configuration'   => json_encode($job['configuration']),
             ];
         }
         DB::table('import_jobs')->insert($insert);

public/js/ff/import/status.js

@@ -0,0 +1,91 @@
+/*
+ * status.js
+ * Copyright (C) 2016 thegrumpydictator@gmail.com
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE file for details.
+ */
+/* globals $, jobImportUrl, jobStartUrl, token  */
+
+
+var startedImport = false;
+var interval = 500;
+$(function () {
+    "use strict";
+
+    // check status, every 500 ms.
+    setTimeout(checkImportStatus, 500);
+
+});
+
+
+function checkImportStatus() {
+    "use strict";
+    $.getJSON(jobImportUrl).success(reportOnJobImport).fail(failedJobImport);
+}
+
+function reportOnJobImport(data) {
+    "use strict";
+    console.log('Now in reportOnJobImport');
+
+    // update bar if it's a percentage or not:
+
+    var bar = $('#import-status-bar');
+    if (data.showPercentage) {
+        console.log('Has percentage.');
+        bar.addClass('progress-bar-success').removeClass('progress-bar-info');
+        bar.attr('aria-valuenow', data.percentage);
+        bar.css('width', data.percentage + '%');
+        $('#import-status-bar').text(data.stepsDone + '/' + data.steps);
+
+
+        if (data.percentage >= 100) {
+            console.log('Now import complete!');
+            bar.removeClass('active');
+            return;
+        }
+
+    } else {
+        $('#import-status-more-info').text('');
+        console.log('Has no percentage.');
+        bar.removeClass('progress-bar-success').addClass('progress-bar-info');
+        bar.attr('aria-valuenow', 100);
+        bar.css('width', '100%');
+    }
+
+    // update the message:
+    $('#import-status-txt').removeClass('text-danger').text(data.statusText);
+
+    // if the job has not actually started, do so now:
+    if (!data.started && !startedImport) {
+        console.log('Will now start job.');
+        $.post(jobStartUrl, {_token: token});
+        startedTheImport();
+        startedImport = true;
+    } else {
+        // trigger another check.
+        setTimeout(checkImportStatus, 500);
+    }
+}
+
+function startedTheImport() {
+    "use strict";
+    console.log('Started the import. Now starting over again.');
+    setTimeout(checkImportStatus, 500);
+}
+
+function failedJobImport(jqxhr, textStatus, error) {
+    "use strict";
+
+    // set status
+    $('#import-status-txt').addClass('text-danger').text(
+        "There was an error during the import routine. Please check the log files. The error seems to be: '" + textStatus + ' ' + error + "'."
+    );
+
+    // remove progress bar.
+    $('#import-status-holder').hide();
+    console.log('failedJobImport');
+    console.log(textStatus);
+    console.log(error);
+
+}

resources/lang/en_US/firefly.php

@@ -767,6 +767,8 @@ return [
     'configure_import'                          => 'Further configure your import',
     'import_finish_configuration'               => 'Finish configuration',
     'settings_for_import'                       => 'Settings',
+    'import_status'                             => 'Import status',
+    'import_status_text'                        => 'The import is currently running, or will start momentarily.',
     'import_complete'                           => 'Import configuration complete!',
     'import_complete_text'                      => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.',
     'import_download_config'                    => 'Download configuration',
@@ -777,4 +779,7 @@ return [
     'import'                                    => 'Import',
     'import_intro_text'                         => 'Welcome to the Firefly III data import routine. At the moment, this routine can help you import files into Firefly. To do so, you must download or export transactions from other systems or software, and upload them here. The next steps will let you help Firefly III determin what the content is of your file, and how to handle it. Please select a file, and read all instructions carefully.',
     'import_file_help'                          => 'Select your file',
+    'import_status_settings_complete'           => 'The import is ready to start.',
+    'import_status_import_complete'             => 'The import has completed.',
+    'import_status_import_running'              => 'The import is currently running. Please be patient. An apparent lack of progress may be a trick of the light.',
 ];

resources/views/import/status.twig

@@ -0,0 +1,47 @@
+{% extends "./layout/default.twig" %}
+
+{% block breadcrumbs %}
+    {{ Breadcrumbs.renderIfExists }}
+{% endblock %}
+{% block content %}
+    <div class="row">
+        <div class="col-lg-12 col-md-12 col-sm-12">
+            <div class="box box-default">
+                <div class="box-header with-border">
+                    <h3 class="box-title">{{ 'import_status'|_ }}</h3>
+                </div>
+                <div class="box-body">
+                    <p>
+                        {{ 'import_status_text'|_ }}
+                    </p>
+                    <div class="row">
+
+                        <div class="col-lg-6 col-lg-offset-3">
+                            <div class="progress" id="import-status-holder">
+                                <div id="import-status-bar" class="progress-bar progress-bar-info active progress-bar-striped" role="progressbar"
+                                     aria-valuenow="100" aria-valuemin="0"
+                                     aria-valuemax="100" style="width: 100%">
+                                </div>
+                            </div>
+                            <p id="import-status-txt"></p>
+                            <p id="import-status-more-info"></p>
+                        </div>
+                    </div>
+
+
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}
+{% block scripts %}
+    <script type="text/javascript">
+        var jobKey = '{{ job.key }}';
+        var jobImportUrl = '{{ route('import.json', [job.key]) }}';
+        var jobStartUrl = '{{ route('import.start', [job.key]) }}';
+        var token = '{{ csrf_token() }}';
+    </script>
+    <script type="text/javascript" src="js/ff/import/status.js"></script>
+{% endblock %}
+{% block styles %}
+{% endblock %}

storage/database/seed.import-test.json

@@ -96,6 +96,10 @@
       "key": "testImport",
       "file_type": "csv",
       "status": "settings_complete",
+      "extended_status": {
+        "steps_done": 0,
+        "total_steps": 0
+      },
       "configuration": {
         "has-headers": false,
         "date-format": "Ymd",