Improve CSV import routine.

app/Import/Routine/FileRoutine.php

@@ -26,7 +26,6 @@ use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
 use FireflyIII\Support\Import\Routine\File\FileProcessorInterface;
-use FireflyIII\Support\Import\Routine\File\FileRoutineInterface;
 use Log;
 
 /**

app/Support/Import/Placeholder/ColumnValue.php

@@ -0,0 +1,113 @@
+<?php
+/**
+ * Column.php
+ * Copyright (c) 2018 thegrumpydictator@gmail.com
+ *
+ * This file is part of Firefly III.
+ *
+ * Firefly III is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Firefly III is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Support\Import\Placeholder;
+
+/**
+ * Class Column
+ */
+class ColumnValue
+{
+    /** @var int */
+    private $mappedValue;
+    /** @var string */
+    private $originalRole;
+    /** @var string */
+    private $role;
+    /** @var string */
+    private $value;
+
+    /**
+     * ColumnValue constructor.
+     */
+    public function __construct()
+    {
+        $this->mappedValue = 0;
+    }
+
+    /**
+     * @return int
+     */
+    public function getMappedValue(): int
+    {
+        return $this->mappedValue;
+    }
+
+    /**
+     * @param int $mappedValue
+     */
+    public function setMappedValue(int $mappedValue): void
+    {
+        $this->mappedValue = $mappedValue;
+    }
+
+    /**
+     * @return string
+     */
+    public function getOriginalRole(): string
+    {
+        return $this->originalRole;
+    }
+
+    /**
+     * @param string $originalRole
+     */
+    public function setOriginalRole(string $originalRole): void
+    {
+        $this->originalRole = $originalRole;
+    }
+
+    /**
+     * @return string
+     */
+    public function getRole(): string
+    {
+        return $this->role;
+    }
+
+    /**
+     * @param string $role
+     */
+    public function setRole(string $role): void
+    {
+        $this->role = $role;
+    }
+
+    /**
+     * @return string
+     */
+    public function getValue(): string
+    {
+        return $this->value;
+    }
+
+    /**
+     * @param string $value
+     */
+    public function setValue(string $value): void
+    {
+        $this->value = $value;
+    }
+
+
+}

app/Support/Import/Placeholder/ImportTransaction.php

@@ -0,0 +1,220 @@
+<?php
+/**
+ * Transaction.php
+ * Copyright (c) 2018 thegrumpydictator@gmail.com
+ *
+ * This file is part of Firefly III.
+ *
+ * Firefly III is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Firefly III is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Support\Import\Placeholder;
+
+use FireflyIII\Exceptions\FireflyException;
+
+/**
+ * Class ImportTransaction
+ */
+class ImportTransaction
+{
+    /** @var string */
+    private $accountBic;
+    /** @var string */
+    private $accountIban;
+    /** @var int */
+    private $accountId;
+    /** @var string */
+    private $accountName;
+    /** @var string */
+    private $amount;
+    /** @var string */
+    private $amountCredit;
+    /** @var string */
+    private $amountDebit;
+    /** @var int */
+    private $billId;
+    /** @var int */
+    private $budgetId;
+    /** @var int */
+    private $categoryId;
+    /** @var int */
+    private $currencyId;
+    /** @var string */
+    private $date;
+    /** @var string */
+    private $description;
+    /** @var string */
+    private $externalId;
+    /** @var string */
+    private $foreignAmount;
+    /** @var int */
+    private $foreignCurrencyId;
+    /** @var array */
+    private $meta;
+    /** @var array */
+    private $modifiers;
+    /** @var string */
+    private $note;
+    /** @var string */
+    private $opposingBic;
+    /** @var string */
+    private $opposingIban;
+    /** @var int */
+    private $opposingId;
+    /** @var string */
+    private $opposingName;
+    /** @var array */
+    private $tags;
+
+    /**
+     * ImportTransaction constructor.
+     */
+    public function __construct()
+    {
+        $this->tags        = [];
+        $this->modifiers   = [];
+        $this->meta        = [];
+        $this->description = '';
+        $this->note        = '';
+    }
+
+    /**
+     * @param ColumnValue $columnValue
+     *
+     * @throws FireflyException
+     */
+    public function addColumnValue(ColumnValue $columnValue): void
+    {
+        switch ($columnValue->getRole()) {
+            default:
+                throw new FireflyException(
+                    sprintf('ImportTransaction cannot handle role "%s" with value "%s"', $columnValue->getRole(), $columnValue->getValue())
+                );
+            case 'account-id':
+                // could be the result of a mapping?
+                $this->accountId = $this->getMappedValue($columnValue);
+                break;
+            case 'account-name':
+                $this->accountName = $columnValue->getValue();
+                break;
+            case 'sepa-ct-id';
+            case 'sepa-ct-op';
+            case 'sepa-db';
+            case 'sepa-cc':
+            case 'sepa-country';
+            case 'sepa-ep';
+            case 'sepa-ci';
+            case 'internal-reference':
+            case 'date-interest':
+            case 'date-invoice':
+            case 'date-book':
+            case 'date-payment':
+            case 'date-process':
+            case 'date-due':
+                $this->meta[$columnValue->getRole()] = $columnValue->getValue();
+                break;
+            case'amount_debit':
+                $this->amountDebit = $columnValue->getValue();
+                break;
+            case'amount_credit':
+                $this->amountCredit = $columnValue->getValue();
+                break;
+            case 'amount':
+                $this->amount = $columnValue->getValue();
+                break;
+            case 'amount_foreign':
+                $this->foreignAmount = $columnValue->getValue();
+                break;
+            case 'foreign-currency-id':
+                $this->foreignCurrencyId = $this->getMappedValue($columnValue);
+                break;
+            case 'bill-id':
+                $this->billId = $this->getMappedValue($columnValue);
+                break;
+            case 'budget-id':
+                $this->budgetId = $this->getMappedValue($columnValue);
+                break;
+            case 'category-id':
+                $this->categoryId = $this->getMappedValue($columnValue);
+                break;
+            case 'currency-id':
+                $this->currencyId = $this->getMappedValue($columnValue);
+                break;
+            case 'date-transaction':
+                $this->date = $columnValue->getValue();
+                break;
+            case 'description':
+                $this->description .= $columnValue->getValue();
+                break;
+            case 'note':
+                $this->note .= $columnValue->getValue();
+                break;
+            case 'external-id':
+                $this->externalId = $columnValue->getValue();
+                break;
+            case 'rabo-debit-credit':
+            case 'ing-debit-credit':
+                $this->modifiers[$columnValue->getRole()] = $columnValue->getValue();
+                break;
+            case 'opposing-id':
+                $this->opposingId = $this->getMappedValue($columnValue);
+                break;
+            case 'opposing-name':
+                $this->opposingName = $columnValue->getValue();
+                break;
+            case 'opposing-bic':
+                $this->opposingBic = $columnValue->getValue();
+                break;
+            case 'tags-comma':
+                // todo split using pre-processor.
+                $this->tags = $columnValue->getValue();
+                break;
+            case 'tags-space':
+                // todo split using pre-processor.
+                $this->tags = $columnValue->getValue();
+                break;
+            case 'account-iban':
+                $this->accountIban = $columnValue->getValue();
+                break;
+            case 'opposing-iban':
+                $this->opposingIban = $columnValue->getValue();
+                break;
+            case '_ignore':
+            case 'bill-name':
+            case 'currency-name':
+            case 'currency-code':
+            case 'foreign-currency-code':
+            case 'currency-symbol':
+            case 'budget-name':
+            case 'category-name':
+            case 'account-number':
+            case 'opposing-number':
+        }
+    }
+
+    /**
+     * Returns the mapped value if it exists in the ColumnValue object.
+     *
+     * @param ColumnValue $columnValue
+     *
+     * @return int
+     */
+    private function getMappedValue(ColumnValue $columnValue): int
+    {
+        return $columnValue->getMappedValue() > 0 ? $columnValue->getMappedValue() : (int)$columnValue->getValue();
+    }
+
+}

app/Support/Import/Routine/File/CSVProcessor.php

@@ -28,6 +28,8 @@ use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
 use FireflyIII\Models\Attachment;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use FireflyIII\Support\Import\Placeholder\ColumnValue;
+use FireflyIII\Support\Import\Placeholder\ImportTransaction;
 use Illuminate\Support\Collection;
 use League\Csv\Exception;
 use League\Csv\Reader;
@@ -44,8 +46,12 @@ class CSVProcessor implements FileProcessorInterface
 {
     /** @var AttachmentHelperInterface */
     private $attachments;
+    /** @var array */
+    private $config;
     /** @var ImportJob */
     private $importJob;
+    /** @var array */
+    private $mappedValues;
     /** @var ImportJobRepositoryInterface */
     private $repository;
 
@@ -57,7 +63,7 @@ class CSVProcessor implements FileProcessorInterface
      */
     public function run(): array
     {
-        $config = $this->importJob->configuration;
+
 
         // in order to actually map we also need to read the FULL file.
         try {
@@ -66,14 +72,11 @@ class CSVProcessor implements FileProcessorInterface
             Log::error($e->getMessage());
             throw new FireflyException('Cannot get reader: ' . $e->getMessage());
         }
-        // get mapping from config
-        $roles = $config['column-roles'];
-
         // get all lines from file:
-        $lines = $this->getLines($reader, $config);
+        $lines = $this->getLines($reader);
 
         // make import objects, according to their role:
-        $importables = $this->processLines($lines, $roles);
+        $importables = $this->processLines($lines);
 
         echo '<pre>';
         print_r($importables);
@@ -89,6 +92,7 @@ class CSVProcessor implements FileProcessorInterface
     public function setJob(ImportJob $job): void
     {
         $this->importJob   = $job;
+        $this->config      = $job->configuration;
         $this->repository  = app(ImportJobRepositoryInterface::class);
         $this->attachments = app(AttachmentHelperInterface::class);
         $this->repository->setUser($job->user);
@@ -99,14 +103,13 @@ class CSVProcessor implements FileProcessorInterface
      * Returns all lines from the CSV file.
      *
      * @param Reader $reader
-     * @param array  $config
      *
      * @return array
      * @throws FireflyException
      */
-    private function getLines(Reader $reader, array $config): array
+    private function getLines(Reader $reader): array
     {
-        $offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
+        $offset = isset($this->config['has-headers']) && $this->config['has-headers'] === true ? 1 : 0;
         try {
             $stmt = (new Statement)->offset($offset);
         } catch (Exception $e) {
@@ -146,45 +149,93 @@ class CSVProcessor implements FileProcessorInterface
     }
 
     /**
-     * Process a single column. Return is an array with:
+     * If the value in the column is mapped to a certain ID,
-     * [0 => key, 1 => value]
+     * the column where this ID must be placed will change.
-     * where the first item is the key under which the value
+     *
-     * must be stored, and the second is the value.
+     * For example, if you map role "budget-name" with value "groceries" to 1,
+     * then that should become the budget-id. Not the name.
      *
      * @param int $column
-     * @param string $value
+     * @param int $mapped
-     * @param string $role
      *
-     * @return array
+     * @return string
      * @throws FireflyException
      */
-    private function processColumn(int $column, string $value, string $role): array
+    private function getRoleForColumn(int $column, int $mapped): string
     {
+        $roles = $this->config['column-roles'];
+        $role  = $roles[$column] ?? '_ignore';
+        if ($mapped === 0) {
+            Log::debug(sprintf('Column #%d with role "%s" is not mapped.', $column, $role));
+
+            return $role;
+        }
+        if (!(isset($this->config['column-do-mapping'][$column]) && $this->config['column-do-mapping'][$column] === true)) {
+
+            return $role;
+        }
         switch ($role) {
             default:
-                throw new FireflyException(sprintf('Cannot handle role "%s" with value "%s"', $role, $value));
+                throw new FireflyException(sprintf('Cannot indicate new role for mapped role "%s"', $role));
-
+            case 'account-id':
-
+            case 'account-name':
-                // feed each line into a new class which will process
+            case 'account-iban':
-            // the line. 
+            case 'account-number':
+                $newRole = 'account-id';
+                break;
+            case 'foreign-currency-id':
+            case 'foreign-currency-code':
+                $newRole = 'foreign-currency-id';
+                break;
+            case 'bill-id':
+            case 'bill-name':
+                $newRole = 'bill-id';
+                break;
+            case 'currency-id':
+            case 'currency-name':
+            case 'currency-code':
+            case 'currency-symbol':
+                $newRole = 'currency-id';
+                break;
+            case 'budget-id':
+            case 'budget-name':
+                $newRole = 'budget-id';
+                break;
+            case 'category-id':
+            case 'category-name':
+                $newRole = 'category-id';
+                break;
+            case 'opposing-id':
+            case 'opposing-name':
+            case 'opposing-iban':
+            case 'opposing-number':
+                $newRole = 'opposing-id';
+                break;
         }
+        Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole));
+
+        // also store the $mapped values in a "mappedValues" array.
+        $this->mappedValues[$newRole] = $mapped;
+
+        return $newRole;
     }
 
+
     /**
      * Process all lines in the CSV file. Each line is processed separately.
      *
      * @param array $lines
-     * @param array $roles
      *
      * @return array
      * @throws FireflyException
      */
-    private function processLines(array $lines, array $roles): array
+    private function processLines(array $lines): array
     {
         $processed = [];
-        foreach ($lines as $line) {
+        $count     = \count($lines);
-            $processed[] = $this->processSingleLine($line, $roles);
+        foreach ($lines as $index => $line) {
-
+            Log::debug(sprintf('Now at line #%d of #%d', $index, $count));
+            $processed[] = $this->processSingleLine($line);
         }
 
         return $processed;
@@ -197,18 +248,34 @@ class CSVProcessor implements FileProcessorInterface
      * @param array $line
      * @param array $roles
      *
-     * @return array
+     * @return ImportTransaction
      * @throws FireflyException
      */
-    private function processSingleLine(array $line, array $roles): array
+    private function processSingleLine(array $line): ImportTransaction
     {
+        $transaction = new ImportTransaction;
         // todo run all specifics on row.
-        $transaction = [];
         foreach ($line as $column => $value) {
             $value        = trim($value);
-            $role  = $roles[$column] ?? '_ignore';
+            $originalRole = $this->config['column-roles'][$column] ?? '_ignore';
-            [$key, $result] = $this->processColumn($column, $value, $role);
+            if (\strlen($value) > 0 && $originalRole !== '_ignore') {
-            // if relevant, find mapped value:
+
+                // is a mapped value present?
+                $mapped = $this->config['column-mapping-config'][$column][$value] ?? 0;
+                // the role might change.
+                $role = $this->getRoleForColumn($column, $mapped);
+
+                $columnValue = new ColumnValue;
+                $columnValue->setValue($value);
+                $columnValue->setRole($role);
+                $columnValue->setMappedValue($mapped);
+                $columnValue->setOriginalRole($originalRole);
+                $transaction->addColumnValue($columnValue);
+
+                // create object that parses this column value.
+
+                Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $role, $value));
+            }
         }
 
         return $transaction;