New code for updated import routine.

app/Http/Controllers/Import/JobConfigurationController.php

@@ -85,7 +85,7 @@ class JobConfigurationController extends Controller
             Log::debug('Job needs no config, is ready to run!');
             $this->repository->updateStatus($importJob ,'ready_to_run');
 
-            return redirect(route('import.job.status.index', [$importProvider->key]));
+            return redirect(route('import.job.status.index', [$importJob->key]));
         }
 
         // create configuration class:

app/Http/Controllers/Import/JobStatusController.php

@@ -22,14 +22,19 @@ declare(strict_types=1);
 
 namespace FireflyIII\Http\Controllers\Import;
 
+use Exception;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Http\Controllers\Controller;
 use FireflyIII\Http\Middleware\IsDemoUser;
 use FireflyIII\Import\Routine\RoutineInterface;
+use FireflyIII\Import\Storage\ImportArrayStorage;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use FireflyIII\Repositories\Tag\TagRepositoryInterface;
 use Illuminate\Http\JsonResponse;
+use Illuminate\Http\RedirectResponse;
 use Log;
+use Symfony\Component\Debug\Exception\FatalThrowableError;
 
 /**
  * Class JobStatusController
@@ -82,14 +87,21 @@ class JobStatusController extends Controller
     }
 
     /**
-     * @param ImportJob $job
+     * @param ImportJob $importJob
      *
      * @return JsonResponse
      */
     public function json(ImportJob $importJob): JsonResponse
     {
-        $json = [
+        $extendedStatus = $importJob->extended_status;
-            'status' => $importJob->status,
+        $json           = [
+            'status'        => $importJob->status,
+            'errors'        => $importJob->errors,
+            'count'         => count($importJob->transactions),
+            'tag_id'        => $importJob->tag_id,
+            'tag_name'      => null === $importJob->tag_id ? null : $importJob->tag->tag,
+            'journals'      => $extendedStatus['count'] ?? 0,
+            'journals_text' => trans_choice('import.status_with_count', $extendedStatus['count'] ?? 0),
         ];
 
         return response()->json($json);
@@ -104,11 +116,11 @@ class JobStatusController extends Controller
     public function start(ImportJob $importJob): JsonResponse
     {
         // catch impossible status:
-        $allowed = ['ready_to_run'];
+        $allowed = ['ready_to_run', 'need_job_config'];
         if (null !== $importJob && !in_array($importJob->status, $allowed)) {
             Log::error('Job is not ready.');
-            session()->flash('error', trans('import.bad_job_status'));
+
-            return redirect(route('import.index'));
+            return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "ready_to_run".']);
         }
 
         $importProvider = $importJob->provider;
@@ -122,6 +134,21 @@ class JobStatusController extends Controller
         // if the job is set to "provider_finished", we should be able to store transactions
         // generated by the provider.
         // otherwise, just continue.
+        if ($importJob->status === 'provider_finished') {
+            try {
+                $this->importFromJob($importJob);
+            } catch (FireflyException $e) {
+                $message = 'The import storage routine crashed: ' . $e->getMessage();
+                Log::error($message);
+                Log::error($e->getTraceAsString());
+
+                // set job errored out:
+                $this->repository->setStatus($importJob, 'error');
+
+                return response()->json(['status' => 'NOK', 'message' => $message]);
+            }
+        }
+
 
         // set job to be running:
         $this->repository->setStatus($importJob, 'running');
@@ -144,6 +171,67 @@ class JobStatusController extends Controller
 
         // expect nothing from routine, just return OK to user.
         return response()->json(['status' => 'OK', 'message' => 'stage_finished']);
+    }
+
+    /**
+     * @param ImportJob $job
+     *
+     * @return JsonResponse
+     * @throws FireflyException
+     */
+    public function store(ImportJob $importJob): JsonResponse
+    {
+        // catch impossible status:
+        $allowed = ['provider_finished', 'storing_data']; // todo remove storing data.
+        if (null !== $importJob && !in_array($importJob->status, $allowed)) {
+            Log::error('Job is not ready.');
+
+            return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "provider_finished".']);
+        }
+
+        // set job to be storing data:
+        $this->repository->setStatus($importJob, 'storing_data');
+
+        try {
+            $this->importFromJob($importJob);
+        } catch (FireflyException $e) {
+            $message = 'The import storage routine crashed: ' . $e->getMessage();
+            Log::error($message);
+            Log::error($e->getTraceAsString());
+
+            // set job errored out:
+            $this->repository->setStatus($importJob, 'error');
+
+            return response()->json(['status' => 'NOK', 'message' => $message]);
+        }
+
+        // set job to be finished.
+        $this->repository->setStatus($importJob, 'finished');
+
+        // expect nothing from routine, just return OK to user.
+        return response()->json(['status' => 'OK', 'message' => 'storage_finished']);
+    }
+
+    /**
+     * @param ImportJob $importJob
+     *
+     * @throws FireflyException
+     */
+    private function importFromJob(ImportJob $importJob): void
+    {
+        try {
+            $storage                 = new ImportArrayStorage($importJob);
+            $journals                = $storage->store();
+            $extendedStatus          = $importJob->extended_status;
+            $extendedStatus['count'] = $journals->count();
+            $this->repository->setExtendedStatus($importJob, $extendedStatus);
+        } catch (FireflyException|Exception|FatalThrowableError $e) {
+            throw new FireflyException($e->getMessage());
+        }
+
+
+
+
     }
 
     //    /**

app/Import/JobConfiguration/FakeJobConfiguration.php

@@ -56,10 +56,16 @@ class FakeJobConfiguration implements JobConfiguratorInterface
         // configuration array of job must have two values:
         // 'artist' must be 'david bowie', case insensitive
         // 'song' must be 'golden years', case insensitive.
+        // if stage is not "new", then album must be 'station to station'
         $config = $this->job->configuration;
+        if ($this->job->stage === 'new') {
+            return (isset($config['artist']) && 'david bowie' === strtolower($config['artist']))
+                   && (isset($config['song']) && 'golden years' === strtolower($config['song']));
+        }
+
+        return isset($config['album']) && 'station to station' === strtolower($config['album']);
+
 
-        return (isset($config['artist']) && 'david bowie' === strtolower($config['artist']))
-               && (isset($config['song']) && 'golden years' === strtolower($config['song']));
     }
 
     /**
@@ -72,16 +78,24 @@ class FakeJobConfiguration implements JobConfiguratorInterface
     public function configureJob(array $data): MessageBag
     {
         $artist        = strtolower($data['artist'] ?? '');
+        $song          = strtolower($data['song'] ?? '');
+        $album         = strtolower($data['album'] ?? '');
         $configuration = $this->job->configuration;
         if ($artist === 'david bowie') {
             // store artist
             $configuration['artist'] = $artist;
         }
-        $song = strtolower($data['song'] ?? '');
+
         if ($song === 'golden years') {
-            // store artist
+            // store song
             $configuration['song'] = $song;
         }
+
+        if ($album=== 'station to station') {
+            // store album
+            $configuration['album'] = $album;
+        }
+
         $this->repository->setConfiguration($this->job, $configuration);
         $messages = new MessageBag();
 
@@ -114,12 +128,16 @@ class FakeJobConfiguration implements JobConfiguratorInterface
         $config = $this->job->configuration;
         $artist = $config['artist'] ?? '';
         $song   = $config['song'] ?? '';
+        $album = $config['album'] ?? '';
         if (strtolower($artist) !== 'david bowie') {
             return 'import.fake.enter-artist';
         }
         if (strtolower($song) !== 'golden years') {
             return 'import.fake.enter-song';
         }
+        if (strtolower($album) !== 'station to station' && $this->job->stage !== 'new') {
+            return 'import.fake.enter-album';
+        }
     }
 
     /**

app/Import/Routine/FakeRoutine.php

@@ -87,9 +87,11 @@ class FakeRoutine implements RoutineInterface
                 break;
             case 'final':
                 $handler = new StageFinalHandler;
+                $handler->setJob($this->job);
                 $transactions = $handler->getTransactions();
                 $this->repository->setStatus($this->job, 'provider_finished');
                 $this->repository->setStage($this->job, 'final');
+                $this->repository->setTransactions($this->job, $transactions);
         }
     }
 

app/Import/Storage/ImportArrayStorage.php

@@ -0,0 +1,361 @@
+<?php
+
+namespace FireflyIII\Import\Storage;
+
+use Carbon\Carbon;
+use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Factory\TransactionJournalFactory;
+use FireflyIII\Helpers\Collector\JournalCollectorInterface;
+use FireflyIII\Helpers\Filter\InternalTransferFilter;
+use FireflyIII\Models\ImportJob;
+use FireflyIII\Models\Tag;
+use FireflyIII\Models\Transaction;
+use FireflyIII\Models\TransactionJournalMeta;
+use FireflyIII\Models\TransactionType;
+use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use FireflyIII\Repositories\Tag\TagRepositoryInterface;
+use Illuminate\Support\Collection;
+use Illuminate\Support\MessageBag;
+use Log;
+use DB;
+
+/**
+ * Creates new transactions based upon arrays. Will first check the array for duplicates.
+ *
+ * Class ImportArrayStorage
+ *
+ * @package FireflyIII\Import\Storage
+ */
+class ImportArrayStorage
+{
+    /** @var bool */
+    private $checkForTransfers = false;
+    /** @var ImportJob */
+    private $importJob;
+
+    /** @var ImportJobRepositoryInterface */
+    private $repository;
+
+    /** @var Collection */
+    private $transfers;
+
+    /**
+     * ImportArrayStorage constructor.
+     *
+     * @param ImportJob $importJob
+     */
+    public function __construct(ImportJob $importJob)
+    {
+        $this->importJob = $importJob;
+        $this->countTransfers();
+
+        $this->repository = app(ImportJobRepositoryInterface::class);
+        $this->repository->setUser($importJob->user);
+
+        Log::debug('Constructed ImportArrayStorage()');
+    }
+
+    /**
+     * Actually does the storing.
+     *
+     * @return Collection
+     * @throws FireflyException
+     */
+    public function store(): Collection
+    {
+        $count = count($this->importJob->transactions);
+        Log::debug(sprintf('Now in store(). Count of items is %d', $count));
+        $toStore = [];
+        foreach ($this->importJob->transactions as $index => $transaction) {
+            Log::debug(sprintf('Now at item %d out of %d', ($index + 1), $count));
+            $existingId = $this->hashExists($transaction);
+            if (null !== $existingId) {
+                $this->logDuplicateObject($transaction, $existingId);
+                $this->repository->addErrorMessage(
+                    $this->importJob, sprintf(
+                                        'Entry #%d ("%s") could not be imported. It already exists.',
+                                        $index, $transaction['description']
+                                    )
+                );
+                continue;
+            }
+            if ($this->checkForTransfers) {
+                if ($this->transferExists($transaction)) {
+                    $this->logDuplicateTransfer($transaction);
+                    $this->repository->addErrorMessage(
+                        $this->importJob, sprintf(
+                                            'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
+                                            $index,
+                                            $transaction['description']
+                                        )
+                    );
+                    continue;
+                }
+            }
+            $toStore[] = $transaction;
+        }
+
+        if (count($toStore) === 0) {
+            Log::info('No transactions to store left!');
+
+            return new Collection;
+        }
+        Log::debug('Going to store...');
+        // now actually store them:
+        $collection = new Collection;
+        /** @var TransactionJournalFactory $factory */
+        $factory = app(TransactionJournalFactory::class);
+        $factory->setUser($this->importJob->user);
+        foreach ($toStore as $store) {
+            // convert the date to an object:
+            $store['date'] = Carbon::createFromFormat('Y-m-d', $store['date']);
+
+            // store the journal.
+            $collection->push($factory->create($store));
+        }
+        Log::debug('DONE storing!');
+
+
+        // create tag and append journals:
+        $this->createTag($collection);
+
+        return $collection;
+    }
+
+    /**
+     * @param Collection $collection
+     */
+    private function createTag(Collection $collection): void
+    {
+
+        /** @var TagRepositoryInterface $repository */
+        $repository = app(TagRepositoryInterface::class);
+        $repository->setUser($this->importJob->user);
+        $data = [
+            'tag'         => trans('import.import_with_key', ['key' => $this->importJob->key]),
+            'date'        => new Carbon,
+            'description' => null,
+            'latitude'    => null,
+            'longitude'   => null,
+            'zoomLevel'   => null,
+            'tagMode'     => 'nothing',
+        ];
+        $tag  = $repository->store($data);
+
+        Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
+        Log::debug('Looping journals...');
+        $journalIds = $collection->pluck('id')->toArray();
+        $tagId      = $tag->id;
+        foreach ($journalIds as $journalId) {
+            Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
+            DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
+        }
+        Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $collection->count(), $tag->id, $tag->tag));
+
+        $this->repository->setTag($this->importJob, $tag);
+
+    }
+
+    /**
+     * Count the number of transfers in the array. If this is zero, don't bother checking for double transfers.
+     */
+    private function countTransfers(): void
+    {
+        $count = 0;
+        foreach ($this->importJob->transactions as $transaction) {
+            if (strtolower(TransactionType::TRANSFER) === $transaction['type']) {
+                $count++;
+            }
+        }
+        $count = 1;
+        if ($count > 0) {
+            $this->checkForTransfers = true;
+
+            // get users transfers. Needed for comparison.
+            $this->getTransfers();
+        }
+
+    }
+
+    /**
+     * Get the users transfers, so they can be compared to whatever the user is trying to import.
+     */
+    private function getTransfers(): void
+    {
+        /** @var JournalCollectorInterface $collector */
+        $collector = app(JournalCollectorInterface::class);
+        $collector->setAllAssetAccounts()
+                  ->setTypes([TransactionType::TRANSFER])
+                  ->withOpposingAccount();
+        $collector->removeFilter(InternalTransferFilter::class);
+        $this->transfers = $collector->getJournals();
+
+    }
+
+
+    /**
+     * @param array $transaction
+     *
+     * @return int|null
+     * @throws FireflyException
+     */
+    private function hashExists(array $transaction): ?int
+    {
+        $json = json_encode($transaction);
+        if ($json === false) {
+            throw new FireflyException('Could not encode import array. Please see the logs.', $transaction);
+        }
+        $hash = hash('sha256', $json, false);
+
+        // find it!
+        /** @var TransactionJournalMeta $entry */
+        $entry = TransactionJournalMeta
+            ::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
+            ->where('data', $hash)
+            ->where('name', 'importHashV2')
+            ->first(['journal_meta.*']);
+        if (null === $entry) {
+            return null;
+        }
+        Log::info(sprintf('Found a transaction journal with an existing hash: %s', $hash));
+
+        return (int)$entry->transaction_journal_id;
+    }
+
+    /**
+     * @param array $transaction
+     * @param int   $existingId
+     */
+    private function logDuplicateObject(array $transaction, int $existingId): void
+    {
+        Log::info(
+            'Transaction is a duplicate, and will not be imported (the hash exists).',
+            [
+                'existing'    => $existingId,
+                'description' => $transaction['description'] ?? '',
+                'amount'      => $transaction['transactions'][0]['amount'] ?? 0,
+                'date'        => isset($transaction['date']) ? $transaction['date'] : '',
+            ]
+        );
+
+    }
+
+    /**
+     * @param array $transaction
+     */
+    private function logDuplicateTransfer(array $transaction): void
+    {
+        Log::info(
+            'Transaction is a duplicate transfer, and will not be imported (such a transfer exists already).',
+            [
+                'description' => $transaction['description'] ?? '',
+                'amount'      => $transaction['transactions'][0]['amount'] ?? 0,
+                'date'        => isset($transaction['date']) ? $transaction['date'] : '',
+            ]
+        );
+    }
+
+    /**
+     * Check if a transfer exists.
+     *
+     * @param $transaction
+     *
+     * @return bool
+     */
+    private function transferExists(array $transaction): bool
+    {
+        Log::debug('Check if is a double transfer.');
+        if (strtolower(TransactionType::TRANSFER) !== $transaction['type']) {
+            Log::debug(sprintf('Is a %s, not a transfer so no.', $transaction['type']));
+
+            return false;
+        }
+
+        // how many hits do we need?
+        $requiredHits = count($transaction['transactions']) * 4;
+        $totalHits    = 0;
+        Log::debug(sprintf('Required hits for transfer comparison is %d', $requiredHits));
+
+        // loop over each split:
+        foreach ($transaction['transactions'] as $current) {
+
+            // get the amount:
+            $amount = (string)($current['amount'] ?? '0');
+            if (bccomp($amount, '0') === -1) {
+                $amount = bcmul($amount, '-1');
+            }
+
+            // get the description:
+            $description = strlen((string)$current['description']) === 0 ? $transaction['description'] : $current['description'];
+
+            // get the source and destination ID's:
+            $currentSourceIDs = [(int)$current['source_id'], (int)$current['destination_id']];
+            sort($currentSourceIDs);
+
+            // get the source and destination names:
+            $currentSourceNames = [(string)$current['source_name'], (string)$current['destination_name']];
+            sort($currentSourceNames);
+
+            // then loop all transfers:
+            /** @var Transaction $transfer */
+            foreach ($this->transfers as $transfer) {
+                // number of hits for this split-transfer combination:
+                $hits = 0;
+                Log::debug(sprintf('Now looking at transaction journal #%d', $transfer->journal_id));
+                // compare amount:
+                Log::debug(sprintf('Amount %s compared to %s', $amount, $transfer->transaction_amount));
+                if (0 !== bccomp($amount, $transfer->transaction_amount)) {
+                    continue;
+                }
+                ++$hits;
+                Log::debug(sprintf('Comparison is a hit! (%s)', $hits));
+
+                // compare description:
+                Log::debug(sprintf('Comparing "%s" to "%s"', $description, $transfer->description));
+                if ($description !== $transfer->description) {
+                    continue;
+                }
+                ++$hits;
+                Log::debug(sprintf('Comparison is a hit! (%s)', $hits));
+
+                // compare date:
+                $transferDate = $transfer->date->format('Y-m-d');
+                Log::debug(sprintf('Comparing dates "%s" to "%s"', $transaction['date'], $transferDate));
+                if ($transaction['date'] !== $transferDate) {
+                    continue;
+                }
+                ++$hits;
+                Log::debug(sprintf('Comparison is a hit! (%s)', $hits));
+
+                // compare source and destination id's
+                $transferSourceIDs = [(int)$transfer->account_id, (int)$transfer->opposing_account_id];
+                sort($transferSourceIDs);
+                Log::debug('Comparing current transaction source+dest IDs', $currentSourceIDs);
+                Log::debug('.. with current transfer source+dest IDs', $transferSourceIDs);
+                if ($currentSourceIDs === $transferSourceIDs) {
+                    ++$hits;
+                    Log::debug(sprintf('Source IDs are the same! (%d)', $hits));
+                }
+                unset($transferSourceIDs);
+
+                // compare source and destination names
+                $transferSource = [(string)$transfer->account_name, (int)$transfer->opposing_account_name];
+                sort($transferSource);
+                Log::debug('Comparing current transaction source+dest names', $currentSourceNames);
+                Log::debug('.. with current transfer source+dest names', $transferSource);
+                if ($currentSourceNames === $transferSource) {
+                    Log::debug(sprintf('Source names are the same! (%d)', $hits));
+                    ++$hits;
+                }
+                $totalHits += $hits;
+                if ($totalHits >= $requiredHits) {
+                    return true;
+                }
+            }
+        }
+        Log::debug(sprintf('Total hits: %d, required: %d', $totalHits, $requiredHits));
+
+        return $totalHits >= $requiredHits;
+    }
+
+}

app/Models/ImportJob.php

@@ -47,9 +47,10 @@ class ImportJob extends Model
             'configuration'   => 'array',
             'extended_status' => 'array',
             'transactions'    => 'array',
+            'errors'          => 'array',
         ];
     /** @var array */
-    protected $fillable = ['key', 'user_id', 'file_type', 'provider', 'status', 'stage', 'configuration', 'extended_status', 'transactions'];
+    protected $fillable = ['key', 'user_id', 'file_type', 'provider', 'status', 'stage', 'configuration', 'extended_status', 'transactions', 'errors'];
 
     /**
      * @param $value
@@ -79,4 +80,13 @@ class ImportJob extends Model
     {
         return $this->belongsTo(User::class);
     }
+
+    /**
+     * @codeCoverageIgnore
+     * @return \Illuminate\Database\Eloquent\Relations\BelongsTo
+     */
+    public function tag()
+    {
+        return $this->belongsTo(Tag::class);
+    }
 }

app/Repositories/ImportJob/ImportJobRepository.php

@@ -25,6 +25,7 @@ namespace FireflyIII\Repositories\ImportJob;
 use Crypt;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\ImportJob;
+use FireflyIII\Models\Tag;
 use FireflyIII\Models\TransactionJournalMeta;
 use FireflyIII\Repositories\User\UserRepositoryInterface;
 use FireflyIII\User;
@@ -126,6 +127,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
                 $importJob = ImportJob::create(
                     [
                         'user_id'         => $this->user->id,
+                        'tag_id'          => null,
                         'provider'        => $importProvider,
                         'file_type'       => '',
                         'key'             => Str::random(12),
@@ -134,6 +136,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
                         'configuration'   => [],
                         'extended_status' => [],
                         'transactions'    => [],
+                        'errors'          => [],
                     ]
                 );
 
@@ -427,4 +430,50 @@ class ImportJobRepository implements ImportJobRepositoryInterface
     {
         return $job->uploadFileContents();
     }
+
+    /**
+     * @param ImportJob $job
+     * @param array     $transactions
+     *
+     * @return ImportJob
+     */
+    public function setTransactions(ImportJob $job, array $transactions): ImportJob
+    {
+        $job->transactions = $transactions;
+        $job->save();
+
+        return $job;
+    }
+
+    /**
+     * Add message to job.
+     *
+     * @param ImportJob $job
+     * @param string    $error
+     *
+     * @return ImportJob
+     */
+    public function addErrorMessage(ImportJob $job, string $error): ImportJob
+    {
+        $errors      = $job->errors;
+        $errors[]    = $error;
+        $job->errors = $errors;
+        $job->save();
+
+        return $job;
+    }
+
+    /**
+     * @param ImportJob $job
+     * @param Tag       $tag
+     *
+     * @return ImportJob
+     */
+    public function setTag(ImportJob $job, Tag $tag): ImportJob
+    {
+        $job->tag()->associate($tag);
+        $job->save();
+
+        return $job;
+    }
 }

app/Repositories/ImportJob/ImportJobRepositoryInterface.php

@@ -23,6 +23,7 @@ declare(strict_types=1);
 namespace FireflyIII\Repositories\ImportJob;
 
 use FireflyIII\Models\ImportJob;
+use FireflyIII\Models\Tag;
 use FireflyIII\User;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
 
@@ -32,6 +33,32 @@ use Symfony\Component\HttpFoundation\File\UploadedFile;
 interface ImportJobRepositoryInterface
 {
 
+    /**
+     * @param ImportJob $job
+     * @param array     $transactions
+     *
+     * @return ImportJob
+     */
+    public function setTransactions(ImportJob $job, array $transactions): ImportJob;
+
+    /**
+     * @param ImportJob $job
+     * @param Tag       $tag
+     *
+     * @return ImportJob
+     */
+    public function setTag(ImportJob $job, Tag $tag): ImportJob;
+
+    /**
+     * Add message to job.
+     *
+     * @param ImportJob $job
+     * @param string    $error
+     *
+     * @return ImportJob
+     */
+    public function addErrorMessage(ImportJob $job, string $error): ImportJob;
+
     /**
      * @param ImportJob $job
      * @param int       $index

app/Repositories/Tag/TagRepository.php

@@ -105,6 +105,7 @@ class TagRepository implements TagRepositoryInterface
     /**
      * @param int $tagId
      *
+     * @deprecated
      * @return Tag
      */
     public function find(int $tagId): Tag
@@ -453,4 +454,14 @@ class TagRepository implements TagRepositoryInterface
 
         return (int)($range[0] + $extra);
     }
+
+    /**
+     * @param int $tagId
+     *
+     * @return Tag|null
+     */
+    public function findNull(int $tagId): ?Tag
+    {
+        return $this->user->tags()->find($tagId);
+    }
 }

app/Repositories/Tag/TagRepositoryInterface.php

@@ -69,6 +69,13 @@ interface TagRepositoryInterface
     /**
      * @param int $tagId
      *
+     * @return Tag|null
+     */
+    public function findNull(int $tagId): ?Tag;
+
+    /**
+     * @param int $tagId
+     * @deprecated
      * @return Tag
      */
     public function find(int $tagId): Tag;

app/Support/Import/Routine/Fake/StageAhoyHandler.php

@@ -36,7 +36,7 @@ class StageAhoyHandler
      */
     public function run(): void
     {
-        for ($i = 0; $i < 15; $i++) {
+        for ($i = 0; $i < 2; $i++) {
             Log::debug(sprintf('Am now in stage AHOY hander, sleeping... (%d)', $i));
             sleep(1);
         }

app/Support/Import/Routine/Fake/StageFinalHandler.php

@@ -2,6 +2,8 @@
 
 namespace FireflyIII\Support\Import\Routine\Fake;
 
+use Carbon\Carbon;
+
 /**
  * Class StageFinalHandler
  *
@@ -9,6 +11,18 @@ namespace FireflyIII\Support\Import\Routine\Fake;
  */
 class StageFinalHandler
 {
+
+    private $job;
+
+    /**
+     * @param mixed $job
+     */
+    public function setJob($job): void
+    {
+        $this->job = $job;
+    }
+
+
     /**
      * @return array
      */
@@ -17,12 +31,92 @@ class StageFinalHandler
         $transactions = [];
 
         for ($i = 0; $i < 5; $i++) {
-            $transaction = [];
+            $transaction = [
+                'type'               => 'withdrawal',
+                'date'               => Carbon::create()->format('Y-m-d'),
+                'tags'               => '',
+                'user'               => $this->job->user_id,
 
+                // all custom fields:
+                'internal_reference' => null,
+                'notes'              => null,
+
+                // journal data:
+                'description'        => 'Some random description #' . random_int(1, 10000),
+                'piggy_bank_id'      => null,
+                'piggy_bank_name'    => null,
+                'bill_id'            => null,
+                'bill_name'          => null,
+
+                // transaction data:
+                'transactions'       => [
+                    [
+                        'currency_id'           => null,
+                        'currency_code'         => 'EUR',
+                        'description'           => null,
+                        'amount'                => random_int(500, 5000) / 100,
+                        'budget_id'             => null,
+                        'budget_name'           => null,
+                        'category_id'           => null,
+                        'category_name'         => null,
+                        'source_id'             => null,
+                        'source_name'           => 'Checking Account',
+                        'destination_id'        => null,
+                        'destination_name'      => 'Random expense account #' . random_int(1, 10000),
+                        'foreign_currency_id'   => null,
+                        'foreign_currency_code' => null,
+                        'foreign_amount'        => null,
+                        'reconciled'            => false,
+                        'identifier'            => 0,
+                    ],
+                ],
+            ];
 
             $transactions[] = $transaction;
         }
 
+        // add a transfer I know exists already
+        $transactions[] = [
+            'type'               => 'transfer',
+            'date'               => '2017-02-28',
+            'tags'               => '',
+            'user'               => $this->job->user_id,
+
+            // all custom fields:
+            'internal_reference' => null,
+            'notes'              => null,
+
+            // journal data:
+            'description'        => 'Saving money for February',
+            'piggy_bank_id'      => null,
+            'piggy_bank_name'    => null,
+            'bill_id'            => null,
+            'bill_name'          => null,
+
+            // transaction data:
+            'transactions'       => [
+                [
+                    'currency_id'           => null,
+                    'currency_code'         => 'EUR',
+                    'description'           => null,
+                    'amount'                => '140',
+                    'budget_id'             => null,
+                    'budget_name'           => null,
+                    'category_id'           => null,
+                    'category_name'         => null,
+                    'source_id'             => 1,
+                    'source_name'           => 'Checking Account',
+                    'destination_id'        => 2,
+                    'destination_name'      => null,
+                    'foreign_currency_id'   => null,
+                    'foreign_currency_code' => null,
+                    'foreign_amount'        => null,
+                    'reconciled'            => false,
+                    'identifier'            => 0,
+                ],
+            ],
+        ];
+
 
         return $transactions;
 

app/Support/Import/Routine/Fake/StageNewHandler.php

@@ -36,7 +36,7 @@ class StageNewHandler
      */
     public function run(): void
     {
-        for ($i = 0; $i < 15; $i++) {
+        for ($i = 0; $i < 2; $i++) {
             Log::debug(sprintf('Am now in stage new hander, sleeping... (%d)', $i));
             sleep(1);
         }

config/import.php

@@ -65,7 +65,7 @@ return [
         'yodlee'  => false,
     ],
     'has_config'    => [
-        'fake'    => true,
+        'fake'    => false,
         'file'    => true,
         'bunq'    => true,
         'spectre' => true,

database/migrations/2018_04_29_174524_changes_for_v474.php

@@ -31,6 +31,10 @@ class ChangesForV474 extends Migration
                 $table->string('provider', 50)->after('file_type')->default('');
                 $table->string('stage', 50)->after('status')->default('');
                 $table->longText('transactions')->after('extended_status');
+                $table->longText('errors')->after('transactions');
+
+                $table->integer('tag_id', false, true)->nullable()->after('user_id');
+                $table->foreign('tag_id')->references('id')->on('tags')->onDelete('set null');
             }
         );
     }

public/js/ff/import/status_v2.js

@@ -22,6 +22,7 @@
 
 var timeOutId;
 var hasStartedJob = false;
+var jobStorageStarted = false;
 var checkInitialInterval = 1000;
 var checkNextInterval = 500;
 var maxLoops = 60;
@@ -59,7 +60,8 @@ function reportOnJobStatus(data) {
             checkOnJob();
             break;
         case "running":
-            showProgressBox();
+        case "storing_data":
+            showProgressBox(data.ttatus);
             checkOnJob();
             break;
 
@@ -67,12 +69,48 @@ function reportOnJobStatus(data) {
             // redirect user to configuration for this job.
             window.location.replace(jobConfigurationUri);
             break;
+        case 'provider_finished':
+            // call routine to store stuff:
+            storeJobData();
+            checkOnJob();
+            break;
+        case "finished":
+            showJobResults(data);
+            break;
         default:
             console.error('Cannot handle status ' + data.status);
 
     }
 }
 
+/**
+ *
+ * @param data
+ */
+function showJobResults(data) {
+    // hide all boxes.
+// hide status boxes:
+    $('.statusbox').hide();
+
+    // render the count:
+    $('#import-status-more-info').append($('<span>').text(data.journals_text));
+
+
+    // render relevant data from JSON thing.
+    if (data.errors.length > 0) {
+        $('#import-status-error-txt').show();
+        data.errors.forEach(function (element) {
+            $('#import-status-errors').append($('<li>').text(element));
+        });
+
+
+    }
+
+    // show success box.
+    $('.status_finished').show();
+
+}
+
 /**
  * Will refresh and get job status.
  */
@@ -100,6 +138,20 @@ function startJob() {
     $.post(jobStartUri, {_token: token}).fail(reportOnSubmitError).done(reportOnSubmit)
 }
 
+/**
+ * Start the storage routine for this job.
+ */
+function storeJobData() {
+    console.log('In storeJobData()');
+    if (jobStorageStarted) {
+        console.log('Store job already started!');
+        return;
+    }
+    console.log('STORAGE JOB STARTED!');
+    jobStorageStarted = true;
+    $.post(jobStorageStartUri, {_token: token}).fail(reportOnSubmitError).done(reportOnSubmit)
+}
+
 /**
  * Function is called when the JSON array could not be retrieved.
  *
@@ -121,12 +173,20 @@ function reportFailure(xhr, status, error) {
     // show error box.
 }
 
-function showProgressBox() {
+/**
+ *
+ */
+function showProgressBox(status) {
     // hide fatal error box:
     $('.fatal_error').hide();
 
     // hide initial status box:
     $('.status_initial').hide();
+    if(status === 'running') {
+        $('#import-status-txt').text(langImportRunning);
+    } else {
+        $('#import-status-txt').text(langImportStoring);
+    }
 
     // show running box:
     $('.status_running').show();

resources/lang/en_US/import.php

@@ -23,36 +23,40 @@ declare(strict_types=1);
 
 return [
     // status of import:
-    'status_wait_title'                    => 'Please hold...',
+    'status_wait_title'          => 'Please hold...',
-    'status_wait_text'                     => 'This box will disappear in a moment.',
+    'status_wait_text'           => 'This box will disappear in a moment.',
-    'status_fatal_title'                   => 'A fatal error occurred',
+    'status_fatal_title'         => 'A fatal error occurred',
-    'status_fatal_text'                    => 'A fatal error occurred, which the import-routine cannot recover from. Please see the explanation in red below.',
+    'status_fatal_text'          => 'A fatal error occurred, which the import-routine cannot recover from. Please see the explanation in red below.',
-    'status_fatal_more'                    => 'If the error is a time-out, the import will have stopped half-way. For some server configurations, it is merely the server that stopped while the import keeps running in the background. To verify this, check out the log files. If the problem persists, consider importing over the command line instead.',
+    'status_fatal_more'          => 'If the error is a time-out, the import will have stopped half-way. For some server configurations, it is merely the server that stopped while the import keeps running in the background. To verify this, check out the log files. If the problem persists, consider importing over the command line instead.',
-    'status_ready_title'                   => 'Import is ready to start',
+    'status_ready_title'         => 'Import is ready to start',
-    'status_ready_text'                    => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.',
+    'status_ready_text'          => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.',
-    'status_ready_noconfig_text'           => 'The import is ready to start. All the configuration you needed to do has been done. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.',
+    'status_ready_noconfig_text' => 'The import is ready to start. All the configuration you needed to do has been done. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.',
-    'status_ready_config'                  => 'Download configuration',
+    'status_ready_config'        => 'Download configuration',
-    'status_ready_start'                   => 'Start the import',
+    'status_ready_start'         => 'Start the import',
-    'status_ready_share'                   => 'Please consider downloading your configuration and sharing it at the <strong><a href="https://github.com/firefly-iii/import-configurations/wiki">import configuration center</a></strong>. This will allow other users of Firefly III to import their files more easily.',
+    'status_ready_share'         => 'Please consider downloading your configuration and sharing it at the <strong><a href="https://github.com/firefly-iii/import-configurations/wiki">import configuration center</a></strong>. This will allow other users of Firefly III to import their files more easily.',
-    'status_job_new'                       => 'The job is brand new.',
+    'status_job_new'             => 'The job is brand new.',
-    'status_job_configuring'               => 'The import is being configured.',
+    'status_job_configuring'     => 'The import is being configured.',
-    'status_job_configured'                => 'The import is configured.',
+    'status_job_configured'      => 'The import is configured.',
-    'status_job_running'                   => 'The import is running.. Please wait..',
+    'status_job_running'         => 'The import is running.. Please wait..',
-    'status_job_error'                     => 'The job has generated an error.',
+    'status_job_storing'         => 'The import is storing your data.. Please wait..',
-    'status_job_finished'                  => 'The import has finished!',
+    'status_job_error'           => 'The job has generated an error.',
-    'status_running_title'                 => 'The import is running',
+    'status_job_finished'        => 'The import has finished!',
-    'status_running_placeholder'           => 'Please hold for an update...',
+    'status_running_title'       => 'The import is running',
-    'status_finished_title'                => 'Import routine finished',
+    'status_running_placeholder' => 'Please hold for an update...',
-    'status_finished_text'                 => 'The import routine has imported your data.',
+    'status_finished_title'      => 'Import routine finished',
-    'status_errors_title'                  => 'Errors during the import',
+    'status_finished_text'       => 'The import routine has imported your data.',
-    'status_errors_single'                 => 'An error has occurred during the import. It does not appear to be fatal.',
+    'status_errors_title'        => 'Errors during the import',
-    'status_errors_multi'                  => 'Some errors occurred during the import. These do not appear to be fatal.',
+    'status_errors_single'       => 'An error has occurred during the import. It does not appear to be fatal.',
+    'status_errors_multi'        => 'Some errors occurred during the import. These do not appear to be fatal.',
+    'status_with_count'          => 'One transaction has been imported|:count transactions have been imported.',
+
     'status_bread_crumb'                   => 'Import status',
     'status_sub_title'                     => 'Import status',
     'config_sub_title'                     => 'Set up your import',
     'status_finished_job'                  => 'The :count transactions imported can be found in tag <a href=":link" class="label label-success" style="font-size:100%;font-weight:normal;">:tag</a>.',
     'status_finished_no_tag'               => 'Firefly III has not collected any transactions from your import file.',
     'import_with_key'                      => 'Import with key \':key\'',
+    'finished_with_errors'                 => 'The import reported some problems.',
 
     // file, upload something
     'file_upload_title'                    => 'Import setup (1/4) - Upload your file',

resources/views/import/fake/enter-album.twig

@@ -0,0 +1,53 @@
+{% extends "./layout/default" %}
+
+{% block breadcrumbs %}
+    {{ Breadcrumbs.render }}
+{% endblock %}
+{% block content %}
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="box">
+                <div class="box-header with-border">
+                    <h3 class="box-title">Enter station for fake import</h3>
+                </div>
+                <div class="box-body">
+                    <p>
+                        Enter "station to station", no matter the capitalization.
+                    </p>
+                </div>
+            </div>
+
+        </div>
+    </div>
+
+    <form method="POST" action="{{ route('import.job.configuration.post', importJob.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
+        <input name="_token" type="hidden" value="{{ csrf_token() }}">
+        <div class="row">
+            <div class="col-lg-12">
+                <div class="box">
+                    <div class="box-header with-border">
+                        <h3 class="box-title">Fields be here.</h3>
+                    </div>
+                    <div class="box-body">
+                        {{ ExpandedForm.text('album') }}
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="row">
+            <div class="col-lg-12">
+                <div class="box">
+                    <div class="box-body">
+                        <button type="submit" class="btn btn-success pull-right">
+                            Submit it!
+                        </button>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </form>
+{% endblock %}
+{% block scripts %}
+{% endblock %}
+{% block styles %}
+{% endblock %}

resources/views/import/status.twig

@@ -56,7 +56,7 @@
                                 <span class="sr-only">Running...</span>
                             </div>
                         </div>
-                        <p id="import-status-txt">Some text here</p>
+                        <p id="import-status-txt"></p>
                     </div>
                 </div>
             </div>
@@ -128,7 +128,7 @@
         </div>
     </div>
     #}
-    {# displays the finished status of the import
+    {# displays the finished status of the import    #}
     <div class="row status_finished statusbox" style="display:none;">
         <div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
             <div class="box box-default">
@@ -138,14 +138,16 @@
                 <div class="box-body">
                     <p id="import-status-intro">
                         {{ trans('import.status_finished_text') }}
+                        <span id="import-status-more-info"></span>
                     </p>
-                    <p id="import-status-more-info"></p>
+                    <p id="import-status-error-txt" style="display:none;">{{ trans('import.finished_with_errors') }}</p>
+                    <ul id="import-status-errors" class="text-danger">
+
+                    </ul>
                 </div>
             </div>
         </div>
     </div>
-    #}
-
     {# box to show error information. #}
     {#
     <div class="row info_errors" style="display:none;">
@@ -171,6 +173,11 @@
         var jobStatusUri = '{{ route('import.job.status.json', [importJob.key]) }}';
         var jobStartUri = '{{ route('import.job.start', [importJob.key]) }}';
         var jobConfigurationUri = '{{ route('import.job.configuration.index', [importJob.key]) }}';
+        var jobStorageStartUri = '{{ route('import.job.store', [importJob.key]) }}';
+
+        // import is running:
+        var langImportRunning = '{{ trans('import.status_job_running') }}';
+        var langImportStoring = '{{ trans('import.status_job_storing') }}';
 
         // some useful translations.
         {#var langImportTimeOutError = '(time out thing)';#}

routes/web.php

@@ -462,6 +462,7 @@ Route::group(
 
     // start the job!
     Route::any('job/start/{importJob}', ['uses' => 'Import\JobStatusController@start', 'as' => 'job.start']);
+    Route::any('job/store/{importJob}', ['uses' => 'Import\JobStatusController@store', 'as' => 'job.store']);
 
     // import method prerequisites:
     #