Towards managing mapping for file imports.

app/Helpers/Attachments/AttachmentHelper.php

@@ -24,6 +24,7 @@ namespace FireflyIII\Helpers\Attachments;
 
 use Crypt;
 use FireflyIII\Models\Attachment;
+use Illuminate\Contracts\Encryption\DecryptException;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Collection;
 use Illuminate\Support\MessageBag;
@@ -64,6 +65,24 @@ class AttachmentHelper implements AttachmentHelperInterface
         $this->uploadDisk    = Storage::disk('upload');
     }
 
+    /**
+     * @param Attachment $attachment
+     *
+     * @return string
+     */
+    public function getAttachmentContent(Attachment $attachment): string
+    {
+        try {
+            $content = Crypt::decrypt($this->uploadDisk->get(sprintf('at-%d.data', $attachment->id)));
+        } catch (DecryptException $e) {
+            Log::error(sprintf('Could not decrypt data of attachment #%d', $attachment->id));
+
+            return '';
+        }
+
+        return $content;
+    }
+
     /**
      * @param Attachment $attachment
      *

app/Helpers/Attachments/AttachmentHelperInterface.php

@@ -39,6 +39,13 @@ interface AttachmentHelperInterface
      */
     public function getAttachmentLocation(Attachment $attachment): string;
 
+    /**
+     * @param Attachment $attachment
+     *
+     * @return string
+     */
+    public function getAttachmentContent(Attachment $attachment): string;
+
     /**
      * @return Collection
      */

app/Http/Controllers/Import/JobStatusController.php

@@ -120,7 +120,7 @@ class JobStatusController extends Controller
         if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
             Log::error('Job is not ready.');
 
-            return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "ready_to_run".']);
+            return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "ready_to_run".']);
         }
 
         $importProvider = $importJob->provider;
@@ -174,7 +174,7 @@ class JobStatusController extends Controller
         if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
             Log::error('Job is not ready.');
 
-            return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "provider_finished".']);
+            return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "provider_finished".']);
         }
 
         // set job to be storing data:

app/Import/JobConfiguration/FileJobConfiguration.php

@@ -27,6 +27,8 @@ use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
 use FireflyIII\Support\Import\Configuration\File\ConfigurationInterface;
+use FireflyIII\Support\Import\Configuration\File\ConfigureMappingHandler;
+use FireflyIII\Support\Import\Configuration\File\ConfigureRolesHandler;
 use FireflyIII\Support\Import\Configuration\File\ConfigureUploadHandler;
 use FireflyIII\Support\Import\Configuration\File\NewFileJobHandler;
 use Illuminate\Support\MessageBag;
@@ -48,6 +50,20 @@ class FileJobConfiguration implements JobConfigurationInterface
     {
     }
 
+    /**
+     * Returns true when the initial configuration for this job is complete.
+     *
+     * @return bool
+     */
+    public function configurationComplete(): bool
+    {
+        if ($this->importJob->stage === 'ready_to_run') {
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * Store any data from the $data array into the job. Anything in the message bag will be flashed
      * as an error to the user, regardless of its content.
@@ -79,6 +95,45 @@ class FileJobConfiguration implements JobConfigurationInterface
         return $configurator->getNextData();
     }
 
+    /**
+     * Returns the view of the next step in the job configuration.
+     *
+     * @throws FireflyException
+     * @return string
+     */
+    public function getNextView(): string
+    {
+        switch ($this->importJob->stage) {
+            case 'new':
+                return 'import.file.new';
+            case 'configure-upload':
+                return 'import.file.configure-upload';
+                break;
+            case 'roles':
+                return 'import.file.roles';
+                break;
+            case 'map':
+                return 'import.file.map';
+                break;
+            default:
+                // @codeCoverageIgnoreStart
+                throw new FireflyException(
+                    sprintf('FileJobConfiguration::getNextView() cannot handle stage "%s"', $this->importJob->stage)
+                );
+            // @codeCoverageIgnoreEnd
+        }
+    }
+
+    /**
+     * @param ImportJob $job
+     */
+    public function setJob(ImportJob $job): void
+    {
+        $this->importJob  = $job;
+        $this->repository = app(ImportJobRepositoryInterface::class);
+        $this->repository->setUser($job->user);
+    }
+
     /**
      * Get the configuration handler for this specific stage.
      *
@@ -95,6 +150,12 @@ class FileJobConfiguration implements JobConfigurationInterface
             case 'configure-upload':
                 $class = ConfigureUploadHandler::class;
                 break;
+            case 'roles':
+                $class = ConfigureRolesHandler::class;
+                break;
+            case 'map':
+                $class = ConfigureMappingHandler::class;
+                break;
             //            case 'upload-config': // has file, needs file config.
             //                $class = UploadConfig::class;
             //                break;
@@ -113,51 +174,4 @@ class FileJobConfiguration implements JobConfigurationInterface
 
         return app($class);
     }
-
-    /**
-     * Returns the view of the next step in the job configuration.
-     *
-     * @throws FireflyException
-     * @return string
-     */
-    public function getNextView(): string
-    {
-        switch ($this->importJob->stage) {
-            case 'new':
-                return 'import.file.new';
-            case 'configure-upload':
-                return 'import.file.configure-upload';
-                break;
-            default:
-                // @codeCoverageIgnoreStart
-                throw new FireflyException(
-                    sprintf('FileJobConfiguration::getNextView() cannot handle stage "%s"', $this->importJob->stage)
-                );
-            // @codeCoverageIgnoreEnd
-        }
-    }
-
-    /**
-     * Returns true when the initial configuration for this job is complete.
-     *
-     * @return bool
-     */
-    public function configurationComplete(): bool
-    {
-        if ($this->importJob->stage === 'ready_to run') {
-            return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * @param ImportJob $job
-     */
-    public function setJob(ImportJob $job): void
-    {
-        $this->importJob  = $job;
-        $this->repository = app(ImportJobRepositoryInterface::class);
-        $this->repository->setUser($job->user);
-    }
 }

app/Support/Import/Configuration/File/ConfigurationInterface.php

@@ -30,6 +30,15 @@ use Illuminate\Support\MessageBag;
  */
 interface ConfigurationInterface
 {
+    /**
+     * Store data associated with current stage.
+     *
+     * @param array $data
+     *
+     * @return MessageBag
+     */
+    public function configureJob(array $data): MessageBag;
+
     /**
      * Get the data necessary to show the configuration screen.
      *
@@ -39,17 +48,6 @@ interface ConfigurationInterface
 
     /**
      * @param ImportJob $job
-     *
-     * @return ConfigurationInterface
      */
-    public function setJob(ImportJob $job);
+    public function setJob(ImportJob $job): void;
-
-    /**
-     * Store data associated with current stage.
-     *
-     * @param array $data
-     *
-     * @return MessageBag
-     */
-    public function configureJob(array $data): MessageBag;
 }

app/Support/Import/Configuration/File/ConfigureMappingHandler.php

@@ -0,0 +1,296 @@
+<?php
+/**
+ * ConfigureMappingHandler.php
+ * Copyright (c) 2018 thegrumpydictator@gmail.com
+ *
+ * This file is part of Firefly III.
+ *
+ * Firefly III is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Firefly III is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Support\Import\Configuration\File;
+
+use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
+use FireflyIII\Import\Mapper\MapperInterface;
+use FireflyIII\Models\Attachment;
+use FireflyIII\Models\ImportJob;
+use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use Illuminate\Support\Collection;
+use Illuminate\Support\MessageBag;
+use League\Csv\Exception;
+use League\Csv\Reader;
+use Log;
+
+/**
+ * Class ConfigureMappingHandler
+ */
+class ConfigureMappingHandler implements ConfigurationInterface
+{
+    /** @var AttachmentHelperInterface */
+    private $attachments;
+    /** @var array */
+    private $columnConfig;
+    /** @var ImportJob */
+    private $importJob;
+    /** @var ImportJobRepositoryInterface */
+    private $repository;
+
+    /**
+     * Store data associated with current stage.
+     *
+     * @param array $data
+     *
+     * @return MessageBag
+     */
+    public function configureJob(array $data): MessageBag
+    {
+        return new MessageBag;
+    }
+
+    /**
+     * Get the data necessary to show the configuration screen.
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    public function getNextData(): array
+    {
+        $config       = $this->importJob->configuration;
+        $columnConfig = $this->doColumnConfig($config);
+
+        // in order to actually map we also need to read the FULL file.
+        try {
+            $reader = $this->getReader();
+        } catch (Exception $e) {
+            Log::error($e->getMessage());
+            throw new FireflyException('Cannot get reader: ' . $e->getMessage());
+        }
+        //
+        //        if ($config['has-headers']) {
+        //            $offset = 1;
+        //        }
+        //        $stmt                 = (new Statement)->offset($offset);
+        //        $results              = $stmt->process($reader);
+        //        $this->validSpecifics = array_keys(config('csv.import_specifics'));
+        //        $indexes              = array_keys($this->data);
+        //        $rowIndex             = 0;
+        //        foreach ($results as $rowIndex => $row) {
+        //            $row = $this->runSpecifics($row);
+        //
+        //            //do something here
+        //            foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
+        //                if (!isset($row[$index])) {
+        //                    // don't really know how to handle this. Just skip, for now.
+        //                    continue;
+        //                }
+        //                $value = trim($row[$index]);
+        //                if (\strlen($value) > 0) {
+        //                    // we can do some preprocessing here,
+        //                    // which is exclusively to fix the tags:
+        //                    if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) {
+        //                        /** @var PreProcessorInterface $preProcessor */
+        //                        $preProcessor                 = app($this->data[$index]['preProcessMap']);
+        //                        $result                       = $preProcessor->run($value);
+        //                        $this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result);
+        //
+        //                        Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
+        //                        Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
+        //                        Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]);
+        //
+        //                        continue;
+        //                    }
+        //
+        //                    $this->data[$index]['values'][] = $value;
+        //                }
+        //            }
+        //        }
+        //        $setIndexes = array_keys($this->data);
+        //        foreach ($setIndexes as $index) {
+        //            $this->data[$index]['values'] = array_unique($this->data[$index]['values']);
+        //            asort($this->data[$index]['values']);
+        //            // if the count of this array is zero, there is nothing to map.
+        //            if (\count($this->data[$index]['values']) === 0) {
+        //                unset($this->data[$index]);
+        //            }
+        //        }
+        //        unset($setIndexes);
+        //
+        //        // save number of rows, thus number of steps, in job:
+        //        $steps                      = $rowIndex * 5;
+        //        $extended                   = $this->job->extended_status;
+        //        $extended['steps']          = $steps;
+        //        $this->job->extended_status = $extended;
+        //        $this->job->save();
+        //
+        //        return $this->data;
+        //         */
+    }
+
+    /**
+     * @param ImportJob $job
+     */
+    public function setJob(ImportJob $job): void
+    {
+        $this->importJob  = $job;
+        $this->repository = app(ImportJobRepositoryInterface::class);
+        $this->repository->setUser($job->user);
+        $this->attachments  = app(AttachmentHelperInterface::class);
+        $this->columnConfig = [];
+    }
+
+    /**
+     * Create the "mapper" class that will eventually return the correct data for the user
+     * to map against. For example: a list of asset accounts. A list of budgets. A list of tags.
+     *
+     * @param string $column
+     *
+     * @return MapperInterface
+     * @throws FireflyException
+     */
+    private function createMapper(string $column): MapperInterface
+    {
+        $mapperClass = config('csv.import_roles.' . $column . '.mapper');
+        $mapperName  = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass);
+        if (!class_exists($mapperName)) {
+            throw new FireflyException(sprintf('Class "%s" does not exist. Cannot map "%s"', $mapperName, $column));
+        }
+
+        return app($mapperName);
+    }
+
+    /**
+     * For each column in the configuration of the job, will:
+     * - validate the role.
+     * - validate if it can be used for mapping
+     * - if so, create an entry in $columnConfig
+     *
+     * @param array $config
+     *
+     * @return array the column configuration.
+     * @throws FireflyException
+     */
+    private function doColumnConfig(array $config): array
+    {
+        /** @var array $requestMapping */
+        $requestMapping = $config['column-do-mapping'] ?? [];
+        $columnConfig   = [];
+        /**
+         * @var int
+         * @var bool $mustBeMapped
+         */
+        foreach ($requestMapping as $index => $requested) {
+            // sanitize column name, so we're sure it's valid.
+            $column    = $this->sanitizeColumnName($config['column-roles'][$index] ?? '_ignore');
+            $doMapping = $this->doMapOfColumn($column, $requested);
+            if ($doMapping) {
+                // user want to map this column. And this is possible.
+                $columnConfig[$index] = [
+                    'name'          => $column,
+                    'options'       => $this->createMapper($column)->getMap(),
+                    'preProcessMap' => $this->getPreProcessorName($column),
+                    'values'        => [],
+                ];
+            }
+        }
+
+        return $columnConfig;
+    }
+
+    /**
+     * For each $name given, and if the user wants to map the column, will return
+     * true when the column can also be mapped.
+     *
+     * Unmappable columns will always return false.
+     * Mappable columns will return $requested.
+     *
+     * @param string $name
+     * @param bool   $requested
+     *
+     * @return bool
+     */
+    private function doMapOfColumn(string $name, bool $requested): bool
+    {
+        $canBeMapped = config('csv.import_roles.' . $name . '.mappable');
+
+        return $canBeMapped && $requested;
+    }
+
+    /**
+     * Will return the name of the pre-processor: a special class that will clean up any input that may be found
+     * in the users input (aka the file uploaded). Only two examples exist at this time: a space or comma separated
+     * list of tags.
+     *
+     * @param string $column
+     *
+     * @return string
+     */
+    private function getPreProcessorName(string $column): string
+    {
+        $name            = '';
+        $hasPreProcess   = config(sprintf('csv.import_roles.%s.pre-process-map', $column));
+        $preProcessClass = config(sprintf('csv.import_roles.%s.pre-process-mapper', $column));
+
+        if (null !== $hasPreProcess && true === $hasPreProcess && null !== $preProcessClass) {
+            $name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
+        }
+
+        return $name;
+    }
+
+    /**
+     * Return an instance of a CSV file reader so content of the file can be read.
+     *
+     * @throws \League\Csv\Exception
+     */
+    private function getReader(): Reader
+    {
+        $content = '';
+        /** @var Collection $collection */
+        $collection = $this->importJob->attachments;
+        /** @var Attachment $attachment */
+        foreach ($collection as $attachment) {
+            if ($attachment->filename === 'import_file') {
+                $content = $this->attachments->getAttachmentContent($attachment);
+                break;
+            }
+        }
+        $config = $this->repository->getConfiguration($this->importJob);
+        $reader = Reader::createFromString($content);
+        $reader->setDelimiter($config['delimiter']);
+
+        return $reader;
+    }
+
+    /**
+     * For each given column name, will return either the name (when it's a valid one)
+     * or return the _ignore column.
+     *
+     * @param string $name
+     *
+     * @return string
+     */
+    private function sanitizeColumnName(string $name): string
+    {
+        /** @var array $validColumns */
+        $validColumns = array_keys(config('csv.import_roles'));
+        if (!\in_array($name, $validColumns, true)) {
+            $name = '_ignore';
+        }
+
+        return $name;
+    }
+}

app/Support/Import/Configuration/File/ConfigureRolesHandler.php

@@ -0,0 +1,394 @@
+<?php
+/**
+ * ConfigureRolesHandler.php
+ * Copyright (c) 2018 thegrumpydictator@gmail.com
+ *
+ * This file is part of Firefly III.
+ *
+ * Firefly III is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Firefly III is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Support\Import\Configuration\File;
+
+use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
+use FireflyIII\Import\Specifics\SpecificInterface;
+use FireflyIII\Models\Attachment;
+use FireflyIII\Models\ImportJob;
+use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use Illuminate\Support\Collection;
+use Illuminate\Support\MessageBag;
+use League\Csv\Exception;
+use League\Csv\Reader;
+use League\Csv\Statement;
+use Log;
+
+/**
+ * Class ConfigureRolesHandler
+ */
+class ConfigureRolesHandler implements ConfigurationInterface
+{
+    /** @var AttachmentHelperInterface */
+    private $attachments;
+    /** @var array */
+    private $examples;
+    /** @var ImportJob */
+    private $importJob;
+    /** @var ImportJobRepositoryInterface */
+    private $repository;
+    /** @var int */
+    private $totalColumns;
+
+    /**
+     * Store data associated with current stage.
+     *
+     * @param array $data
+     *
+     * @return MessageBag
+     */
+    public function configureJob(array $data): MessageBag
+    {
+        $config = $this->importJob->configuration;
+        $count  = $config['column-count'];
+        for ($i = 0; $i < $count; ++$i) {
+            $role                            = $data['role'][$i] ?? '_ignore';
+            $mapping                         = (isset($data['map'][$i]) && $data['map'][$i] === '1');
+            $config['column-roles'][$i]      = $role;
+            $config['column-do-mapping'][$i] = $mapping;
+            Log::debug(sprintf('Column %d has been given role %s (mapping: %s)', $i, $role, var_export($mapping, true)));
+        }
+        $config   = $this->ignoreUnmappableColumns($config);
+        $messages = $this->configurationComplete($config);
+
+        if ($messages->count() === 0) {
+            $this->repository->setStage($this->importJob, 'ready_to_run');
+            if ($this->isMappingNecessary($config)) {
+                $this->repository->setStage($this->importJob, 'map');
+            }
+            $this->repository->setConfiguration($this->importJob, $config);
+        }
+
+        return $messages;
+    }
+
+    /**
+     * Get the data necessary to show the configuration screen.
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    public function getNextData(): array
+    {
+        try {
+            $reader = $this->getReader();
+        } catch (Exception $e) {
+            Log::error($e->getMessage());
+            throw new FireflyException($e->getMessage());
+        }
+        $headers = $this->getHeaders($reader);
+
+        // get example rows:
+        $this->getExamples($reader);
+
+        return [
+            'examples' => $this->examples,
+            'roles'    => $this->getRoles(),
+            'total'    => $this->totalColumns,
+            'headers'  => $headers,
+        ];
+    }
+
+    /**
+     * Set job and some start values.
+     *
+     * @param ImportJob $job
+     */
+    public function setJob(ImportJob $job): void
+    {
+        $this->importJob  = $job;
+        $this->repository = app(ImportJobRepositoryInterface::class);
+        $this->repository->setUser($job->user);
+        $this->attachments  = app(AttachmentHelperInterface::class);
+        $this->totalColumns = 0;
+        $this->examples     = [];
+    }
+
+    /**
+     * Verifies that the configuration of the job is actually complete, and valid.
+     *
+     * @param array $config
+     *
+     * @return MessageBag
+     */
+    private function configurationComplete(array $config): MessageBag
+    {
+        $count    = $config['column-count'];
+        $assigned = 0;
+
+        // check if data actually contains amount column (foreign amount does not count)
+        $hasAmount        = false;
+        $hasForeignAmount = false;
+        $hasForeignCode   = false;
+        for ($i = 0; $i < $count; ++$i) {
+            $role = $config['column-roles'][$i] ?? '_ignore';
+            if ('_ignore' !== $role) {
+                ++$assigned;
+            }
+            if (\in_array($role, ['amount', 'amount_credit', 'amount_debit'])) {
+                $hasAmount = true;
+            }
+            if ($role === 'foreign-currency-code') {
+                $hasForeignCode = true;
+            }
+            if ($role === 'amount_foreign') {
+                $hasForeignAmount = true;
+            }
+        }
+
+        // all assigned and correct foreign info
+        if ($assigned > 0 && $hasAmount && ($hasForeignCode === $hasForeignAmount)) {
+            return new MessageBag;
+        }
+        if (0 === $assigned || !$hasAmount) {
+            $message  = (string)trans('import.job_config_roles_rwarning');
+            $messages = new MessageBag();
+            $messages->add('error', $message);
+
+            return $messages;
+        }
+
+        // warn if has foreign amount but no currency code:
+        if ($hasForeignAmount && !$hasForeignCode) {
+            $message  = (string)trans('import.job_config_roles_fa_warning');
+            $messages = new MessageBag();
+            $messages->add('error', $message);
+
+            return $messages;
+        }
+
+
+
+        return new MessageBag;
+    }
+
+    /**
+     * Extracts example data from a single row and store it in the class.
+     *
+     * @param array $row
+     */
+    private function getExampleFromRow(array $row): void
+    {
+        foreach ($row as $index => $value) {
+            $value = trim($value);
+            if (\strlen($value) > 0) {
+                $this->examples[$index][] = $value;
+            }
+        }
+    }
+
+    /**
+     * Return a bunch of examples from the CSV file the user has uploaded.
+     *
+     * @param Reader $reader
+     *
+     * @throws FireflyException
+     */
+    private function getExamples(Reader $reader): void
+    {
+        // configure example data:
+        $config = $this->importJob->configuration;
+        $limit  = (int)config('csv.example_rows', 5);
+        $offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
+
+        // make statement.
+        try {
+            $stmt = (new Statement)->limit($limit)->offset($offset);
+        } catch (Exception $e) {
+            Log::error($e->getMessage());
+            throw new FireflyException($e->getMessage());
+        }
+
+        // grab the records:
+        $records = $stmt->process($reader);
+        /** @var array $row */
+        foreach ($records as $row) {
+            $row                = array_values($row);
+            $row                = $this->processSpecifics($row);
+            $count              = \count($row);
+            $this->totalColumns = $count > $this->totalColumns ? $count : $this->totalColumns;
+            $this->getExampleFromRow($row);
+        }
+        // save column count:
+        $this->saveColumCount();
+        $this->makeExamplesUnique();
+    }
+
+    /**
+     * Get the header row, if one is present.
+     *
+     * @param Reader $reader
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function getHeaders(Reader $reader): array
+    {
+        $headers = [];
+        $config  = $this->importJob->configuration;
+        if ($config['has-headers']) {
+            try {
+                $stmt    = (new Statement)->limit(1)->offset(0);
+                $records = $stmt->process($reader);
+                $headers = $records->fetchOne(0);
+            } catch (Exception $e) {
+                Log::error($e->getMessage());
+                throw new FireflyException($e->getMessage());
+            }
+            Log::debug('Detected file headers:', $headers);
+        }
+
+        return $headers;
+    }
+
+    /**
+     * Return an instance of a CSV file reader so content of the file can be read.
+     *
+     * @throws \League\Csv\Exception
+     */
+    private function getReader(): Reader
+    {
+        $content = '';
+        /** @var Collection $collection */
+        $collection = $this->importJob->attachments;
+        /** @var Attachment $attachment */
+        foreach ($collection as $attachment) {
+            if ($attachment->filename === 'import_file') {
+                $content = $this->attachments->getAttachmentContent($attachment);
+                break;
+            }
+        }
+        $config = $this->repository->getConfiguration($this->importJob);
+        $reader = Reader::createFromString($content);
+        $reader->setDelimiter($config['delimiter']);
+
+        return $reader;
+    }
+
+    /**
+     * Returns all possible roles and translate their name. Then sort them.
+     *
+     * @return array
+     */
+    private function getRoles(): array
+    {
+        $roles = [];
+        foreach (array_keys(config('csv.import_roles')) as $role) {
+            $roles[$role] = trans('import.column_' . $role);
+        }
+        asort($roles);
+
+        return $roles;
+    }
+
+    /**
+     * If the user has checked columns that cannot be mapped to any value, this function will
+     * uncheck them and return the configuration again.
+     *
+     * @param array $config
+     *
+     * @return array
+     */
+    private function ignoreUnmappableColumns(array $config): array
+    {
+        $count = $config['column-count'];
+        for ($i = 0; $i < $count; ++$i) {
+            $role    = $config['column-roles'][$i] ?? '_ignore';
+            $mapping = $config['column-do-mapping'][$i] ?? false;
+            // if the column can be mapped depends on the config:
+            $canMap                          = (bool)config(sprintf('csv.import_roles.%s.mappable', $role));
+            $mapping                         = $mapping && $canMap;
+            $config['column-do-mapping'][$i] = $mapping;
+        }
+
+        return $config;
+    }
+
+    /**
+     * Returns false when it's not necessary to map values. This saves time and is user friendly
+     * (will skip to the next screen).
+     *
+     * @param array $config
+     *
+     * @return bool
+     */
+    private function isMappingNecessary(array $config): bool
+    {
+        $count      = $config['column-count'];
+        $toBeMapped = 0;
+        for ($i = 0; $i < $count; ++$i) {
+            $mapping = $config['column-do-mapping'][$i] ?? false;
+            if (true === $mapping) {
+                ++$toBeMapped;
+            }
+        }
+
+        return !(0 === $toBeMapped);
+    }
+
+    /**
+     * Make sure that the examples do not contain double data values.
+     */
+    private function makeExamplesUnique(): void
+    {
+        foreach ($this->examples as $index => $values) {
+            $this->examples[$index] = array_unique($values);
+        }
+    }
+
+    /**
+     * if the user has configured specific fixes to be applied, they must be applied to the example data as well.
+     *
+     * @param array $row
+     *
+     * @return array
+     */
+    private function processSpecifics(array $row): array
+    {
+        $config    = $this->importJob->configuration;
+        $specifics = $config['specifics'] ?? [];
+        $names     = array_keys($specifics);
+        foreach ($names as $name) {
+            /** @var SpecificInterface $specific */
+            $specific = app('FireflyIII\Import\Specifics\\' . $name);
+            $row      = $specific->run($row);
+        }
+
+        return $row;
+
+    }
+
+    /**
+     * Save the column count in the job. It's used in a later stage.
+     *
+     * @return void
+     */
+    private function saveColumCount(): void
+    {
+        $config                 = $this->importJob->configuration;
+        $config['column-count'] = $this->totalColumns;
+        $this->repository->setConfiguration($this->importJob, $config);
+    }
+}

app/Support/Import/Configuration/File/ConfigureUploadHandler.php

@@ -83,10 +83,8 @@ class ConfigureUploadHandler implements ConfigurationInterface
 
     /**
      * @param ImportJob $job
-     *
-     * @return ConfigurationInterface
      */
-    public function setJob(ImportJob $job)
+    public function setJob(ImportJob $job): void
     {
         $this->importJob  = $job;
         $this->repository = app(ImportJobRepositoryInterface::class);

resources/lang/en_US/import.php

@@ -98,7 +98,7 @@ return [
     'job_config_uc_apply_rules_title'    => 'Apply rules',
     'job_config_uc_apply_rules_text'     => 'Applies your rules to every imported transaction. Note that this slows the import significantly.',
     'job_config_uc_specifics_title'      => 'Bank-specific options',
-    'job_config_uc_specifics_txt'        => 'Some banks deliver badly formatted files. Firefly III can fix those automatically. If your bank delivers such files, open an issue on GitHub.',
+    'job_config_uc_specifics_txt'        => 'Some banks deliver badly formatted files. Firefly III can fix those automatically. If your bank delivers such files but it\'s not listed here, please open an issue on GitHub.',
     'job_config_uc_submit'               => 'Continue',
     'invalid_import_account'             => 'You have selected an invalid account to import into.',
     // specifics:
@@ -112,6 +112,18 @@ return [
     'specific_rabo_descr'                => 'Fixes potential problems with Rabobank files',
     'specific_pres_name'                 => 'President\'s Choice Financial CA',
     'specific_pres_descr'                => 'Fixes potential problems with PC files',
+    // job configuration for file provider (stage: roles)
+    'job_config_roles_title'             => 'Import setup (3/4) - Define each column\'s role',
+    'job_config_roles_text'              => 'Each column in your CSV file contains certain data. Please indicate what kind of data the importer should expect. The option to "map" data means that you will link each entry found in the column to a value in your database. An often mapped column is the column that contains the IBAN of the opposing account. That can be easily matched to IBAN\'s present in your database already.',
+    'job_config_roles_submit'            => 'Continue',
+    'job_config_roles_column_name'       => 'Name of column',
+    'job_config_roles_column_example'    => 'Column example data',
+    'job_config_roles_column_role'       => 'Column data meaning',
+    'job_config_roles_do_map_value'      => 'Map these values',
+    'job_config_roles_no_example'        => 'No example data available',
+    'job_config_roles_fa_warning'        => 'If you mark a column as containing an amount in a foreign currency, you must also set the column that contains which currency it is.',
+    'job_config_roles_rwarning'          => 'At the very least, mark one column as the amount-column. It is advisable to also select a column for the description, date and the opposing account.',
+    'job_config_roles_colum_count'       => 'Column',
 
 
     // import status page:
@@ -137,6 +149,55 @@ return [
     // general errors and warnings:
     'bad_job_status'                     => 'To access this page, your import job cannot have status ":status".',
 
+    // column roles for CSV import:
+    'column__ignore'                     => '(ignore this column)',
+    'column_account-iban'                => 'Asset account (IBAN)',
+    'column_account-id'                  => 'Asset account ID (matching FF3)',
+    'column_account-name'                => 'Asset account (name)',
+    'column_amount'                      => 'Amount',
+    'column_amount_foreign'              => 'Amount (in foreign currency)',
+    'column_amount_debit'                => 'Amount (debit column)',
+    'column_amount_credit'               => 'Amount (credit column)',
+    'column_amount-comma-separated'      => 'Amount (comma as decimal separator)',
+    'column_bill-id'                     => 'Bill ID (matching FF3)',
+    'column_bill-name'                   => 'Bill name',
+    'column_budget-id'                   => 'Budget ID (matching FF3)',
+    'column_budget-name'                 => 'Budget name',
+    'column_category-id'                 => 'Category ID (matching FF3)',
+    'column_category-name'               => 'Category name',
+    'column_currency-code'               => 'Currency code (ISO 4217)',
+    'column_foreign-currency-code'       => 'Foreign currency code (ISO 4217)',
+    'column_currency-id'                 => 'Currency ID (matching FF3)',
+    'column_currency-name'               => 'Currency name (matching FF3)',
+    'column_currency-symbol'             => 'Currency symbol (matching FF3)',
+    'column_date-interest'               => 'Interest calculation date',
+    'column_date-book'                   => 'Transaction booking date',
+    'column_date-process'                => 'Transaction process date',
+    'column_date-transaction'            => 'Date',
+    'column_date-due'                    => 'Transaction due date',
+    'column_date-payment'                => 'Transaction payment date',
+    'column_date-invoice'                => 'Transaction invoice date',
+    'column_description'                 => 'Description',
+    'column_opposing-iban'               => 'Opposing account (IBAN)',
+    'column_opposing-bic'                => 'Opposing account (BIC)',
+    'column_opposing-id'                 => 'Opposing account ID (matching FF3)',
+    'column_external-id'                 => 'External ID',
+    'column_opposing-name'               => 'Opposing account (name)',
+    'column_rabo-debit-credit'           => 'Rabobank specific debit/credit indicator',
+    'column_ing-debit-credit'            => 'ING specific debit/credit indicator',
+    'column_sepa-ct-id'                  => 'SEPA end-to-end Identifier',
+    'column_sepa-ct-op'                  => 'SEPA Opposing Account Identifier',
+    'column_sepa-db'                     => 'SEPA Mandate Identifier',
+    'column_sepa-cc'                     => 'SEPA Clearing Code',
+    'column_sepa-ci'                     => 'SEPA Creditor Identifier',
+    'column_sepa-ep'                     => 'SEPA External Purpose',
+    'column_sepa-country'                => 'SEPA Country Code',
+    'column_tags-comma'                  => 'Tags (comma separated)',
+    'column_tags-space'                  => 'Tags (space separated)',
+    'column_account-number'              => 'Asset account (account number)',
+    'column_opposing-number'             => 'Opposing account (account number)',
+    'column_note'                        => 'Note(s)',
+    'column_internal-reference'          => 'Internal reference',
 
     // status of import:
     //    'status_wait_title'          => 'Please hold...',

resources/views/import/file/map.twig

@@ -1,7 +1,7 @@
 {% extends "./layout/default" %}
 
 {% block breadcrumbs %}
-    {{ Breadcrumbs.render(Route.getCurrentRoute.getName, job) }}
+    {{ Breadcrumbs.render(Route.getCurrentRoute.getName, importJob) }}
 {% endblock %}
 
 {% block content %}
@@ -27,7 +27,7 @@
 
         </div>
     </div>
-    <form action="{{ route('import.configure.post', job.key) }}" method="post">
+    <form action="{{ route('import.job.configuration.post', importJob.key) }}" method="post">
         <input type="hidden" name="_token" value="{{ csrf_token() }}"/>
 
 

resources/views/import/file/roles.twig

@@ -1,7 +1,7 @@
 {% extends "./layout/default" %}
 
 {% block breadcrumbs %}
-    {{ Breadcrumbs.render(Route.getCurrentRoute.getName, job) }}
+    {{ Breadcrumbs.render(Route.getCurrentRoute.getName, importJob) }}
 {% endblock %}
 
 {% block content %}
@@ -10,36 +10,34 @@
         <div class="col-lg-12">
             <div class="box">
                 <div class="box-header with-border">
-                    <h3 class="box-title">{{ trans('import.csv_roles_title') }}</h3>
+                    <h3 class="box-title">{{ trans('import.job_config_roles_title') }}</h3>
                 </div>
                 <div class="box-body">
                     <p>
-                        {{ trans('import.csv_roles_text') }}
+                        {{ trans('import.job_config_roles_text') }}
                     </p>
                 </div>
             </div>
 
         </div>
     </div>
-    <form action="{{ route('import.configure.post', job.key) }}" method="post">
+    <form action="{{ route('import.job.configuration.post', importJob.key) }}" method="post">
         <input type="hidden" name="_token" value="{{ csrf_token() }}"/>
-        <input type="hidden" name="settings" value="roles"/>
 
         <div class="row">
             <div class="col-lg-12">
                 <div class="box">
                     <div class="box-header with-border">
-                        <h3 class="box-title">{{ trans('import.csv_roles_table') }}</h3>
+                        <h3 class="box-title">{{ trans('import.job_config_input') }}</h3>
                     </div>
                     <div class="box-body">
-
                         <table class="table">
                             <thead>
                             <tr>
-                                <th style="width:20%;">{{ trans('import.csv_roles_column_name') }}</th>
+                                <th style="width:20%;">{{ trans('import.job_config_roles_column_name') }}</th>
-                                <th style="width:40%;">{{ trans('import.csv_roles_column_example') }}</th>
+                                <th style="width:40%;">{{ trans('import.job_config_roles_column_example') }}</th>
-                                <th style="width:30%;">{{ trans('import.csv_roles_column_role') }}</th>
+                                <th style="width:30%;">{{ trans('import.job_config_roles_column_role') }}</th>
-                                <th style="width:10%;">{{ trans('import.csv_roles_do_map_value') }}</th>
+                                <th style="width:10%;">{{ trans('import.job_config_roles_do_map_value') }}</th>
                             </tr>
                             </thead>
                             {% for i in 0..(data.total -1) %}
@@ -47,14 +45,14 @@
                                 <tr>
                                     <td>
                                         {% if data.headers[i] == '' %}
-                                            {{ trans('import.csv_roles_column') }} #{{ loop.index }}
+                                            {{ trans('import.job_config_roles_colum_count') }} #{{ loop.index }}
                                         {% else %}
                                             {{ data.headers[i] }}
                                         {% endif %}
                                     </td>
                                     <td>
                                         {% if data.examples[i]|length == 0 %}
-                                            <em>{{ trans('import.csv_roles_no_example_data') }}</em>
+                                            <em>{{ trans('import.job_config_roles_no_example') }}</em>
                                         {% else %}
                                             {% for example in data.examples[i] %}
                                                 <code>{{ example }}</code><br/>
@@ -64,12 +62,12 @@
                                     <td>
                                         {{ Form.select(('role['~loop.index0~']'),
                                         data.roles,
-                                        job.configuration['column-roles'][loop.index0],
+                                        importJob.configuration['column-roles'][loop.index0],
                                         {class: 'form-control'}) }}
                                     </td>
                                     <td>
                                         {{ Form.checkbox(('map['~loop.index0~']'),1,
-                                        job.configuration['column-do-mapping'][loop.index0]
+                                        importJob.configuration['column-do-mapping'][loop.index0]
 
                                         ) }}
                                     </td>
@@ -80,7 +78,6 @@
 
                         </table>
 
-
                     </div>
                 </div>
             </div>
@@ -91,7 +88,7 @@
                 <div class="box">
                     <div class="box-body">
                         <button type="submit" class="btn btn-success pull-right">
-                            {{ trans('import.csv_roles_submit') }} <i class="fa fa-arrow-right"></i>
+                            {{ trans('import.job_config_roles_submit') }} <i class="fa fa-arrow-right"></i>
                         </button>
                     </div>
                 </div>