From a3cbdadb397364a184b55da373e425004ae6f9c4 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sun, 6 May 2018 16:19:29 +0200 Subject: [PATCH] Towards managing mapping for file imports. --- app/Helpers/Attachments/AttachmentHelper.php | 19 + .../Attachments/AttachmentHelperInterface.php | 7 + .../Import/JobStatusController.php | 4 +- .../JobConfiguration/FileJobConfiguration.php | 108 ++--- .../File/ConfigurationInterface.php | 22 +- .../File/ConfigureMappingHandler.php | 296 +++++++++++++ .../File/ConfigureRolesHandler.php | 394 ++++++++++++++++++ .../File/ConfigureUploadHandler.php | 4 +- resources/lang/en_US/import.php | 63 ++- resources/views/import/file/map.twig | 4 +- resources/views/import/file/roles.twig | 31 +- 11 files changed, 868 insertions(+), 84 deletions(-) create mode 100644 app/Support/Import/Configuration/File/ConfigureMappingHandler.php create mode 100644 app/Support/Import/Configuration/File/ConfigureRolesHandler.php diff --git a/app/Helpers/Attachments/AttachmentHelper.php b/app/Helpers/Attachments/AttachmentHelper.php index b4da01386b..906b7e39fe 100644 --- a/app/Helpers/Attachments/AttachmentHelper.php +++ b/app/Helpers/Attachments/AttachmentHelper.php @@ -24,6 +24,7 @@ namespace FireflyIII\Helpers\Attachments; use Crypt; use FireflyIII\Models\Attachment; +use Illuminate\Contracts\Encryption\DecryptException; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Collection; use Illuminate\Support\MessageBag; @@ -64,6 +65,24 @@ class AttachmentHelper implements AttachmentHelperInterface $this->uploadDisk = Storage::disk('upload'); } + /** + * @param Attachment $attachment + * + * @return string + */ + public function getAttachmentContent(Attachment $attachment): string + { + try { + $content = Crypt::decrypt($this->uploadDisk->get(sprintf('at-%d.data', $attachment->id))); + } catch (DecryptException $e) { + Log::error(sprintf('Could not decrypt data of attachment #%d', $attachment->id)); + + return ''; + } + + return $content; + } + /** * @param Attachment $attachment * diff --git a/app/Helpers/Attachments/AttachmentHelperInterface.php b/app/Helpers/Attachments/AttachmentHelperInterface.php index fc66698573..276d312a27 100644 --- a/app/Helpers/Attachments/AttachmentHelperInterface.php +++ b/app/Helpers/Attachments/AttachmentHelperInterface.php @@ -39,6 +39,13 @@ interface AttachmentHelperInterface */ public function getAttachmentLocation(Attachment $attachment): string; + /** + * @param Attachment $attachment + * + * @return string + */ + public function getAttachmentContent(Attachment $attachment): string; + /** * @return Collection */ diff --git a/app/Http/Controllers/Import/JobStatusController.php b/app/Http/Controllers/Import/JobStatusController.php index 7faeae86e5..fd5e8127b8 100644 --- a/app/Http/Controllers/Import/JobStatusController.php +++ b/app/Http/Controllers/Import/JobStatusController.php @@ -120,7 +120,7 @@ class JobStatusController extends Controller if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) { Log::error('Job is not ready.'); - return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "ready_to_run".']); + return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "ready_to_run".']); } $importProvider = $importJob->provider; @@ -174,7 +174,7 @@ class JobStatusController extends Controller if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) { Log::error('Job is not ready.'); - return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "provider_finished".']); + return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "provider_finished".']); } // set job to be storing data: diff --git a/app/Import/JobConfiguration/FileJobConfiguration.php b/app/Import/JobConfiguration/FileJobConfiguration.php index 0cfc8c37f1..27eaa21e0a 100644 --- a/app/Import/JobConfiguration/FileJobConfiguration.php +++ b/app/Import/JobConfiguration/FileJobConfiguration.php @@ -27,6 +27,8 @@ use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Support\Import\Configuration\File\ConfigurationInterface; +use FireflyIII\Support\Import\Configuration\File\ConfigureMappingHandler; +use FireflyIII\Support\Import\Configuration\File\ConfigureRolesHandler; use FireflyIII\Support\Import\Configuration\File\ConfigureUploadHandler; use FireflyIII\Support\Import\Configuration\File\NewFileJobHandler; use Illuminate\Support\MessageBag; @@ -48,6 +50,20 @@ class FileJobConfiguration implements JobConfigurationInterface { } + /** + * Returns true when the initial configuration for this job is complete. + * + * @return bool + */ + public function configurationComplete(): bool + { + if ($this->importJob->stage === 'ready_to_run') { + return true; + } + + return false; + } + /** * Store any data from the $data array into the job. Anything in the message bag will be flashed * as an error to the user, regardless of its content. @@ -79,6 +95,45 @@ class FileJobConfiguration implements JobConfigurationInterface return $configurator->getNextData(); } + /** + * Returns the view of the next step in the job configuration. + * + * @throws FireflyException + * @return string + */ + public function getNextView(): string + { + switch ($this->importJob->stage) { + case 'new': + return 'import.file.new'; + case 'configure-upload': + return 'import.file.configure-upload'; + break; + case 'roles': + return 'import.file.roles'; + break; + case 'map': + return 'import.file.map'; + break; + default: + // @codeCoverageIgnoreStart + throw new FireflyException( + sprintf('FileJobConfiguration::getNextView() cannot handle stage "%s"', $this->importJob->stage) + ); + // @codeCoverageIgnoreEnd + } + } + + /** + * @param ImportJob $job + */ + public function setJob(ImportJob $job): void + { + $this->importJob = $job; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->repository->setUser($job->user); + } + /** * Get the configuration handler for this specific stage. * @@ -95,6 +150,12 @@ class FileJobConfiguration implements JobConfigurationInterface case 'configure-upload': $class = ConfigureUploadHandler::class; break; + case 'roles': + $class = ConfigureRolesHandler::class; + break; + case 'map': + $class = ConfigureMappingHandler::class; + break; // case 'upload-config': // has file, needs file config. // $class = UploadConfig::class; // break; @@ -113,51 +174,4 @@ class FileJobConfiguration implements JobConfigurationInterface return app($class); } - - /** - * Returns the view of the next step in the job configuration. - * - * @throws FireflyException - * @return string - */ - public function getNextView(): string - { - switch ($this->importJob->stage) { - case 'new': - return 'import.file.new'; - case 'configure-upload': - return 'import.file.configure-upload'; - break; - default: - // @codeCoverageIgnoreStart - throw new FireflyException( - sprintf('FileJobConfiguration::getNextView() cannot handle stage "%s"', $this->importJob->stage) - ); - // @codeCoverageIgnoreEnd - } - } - - /** - * Returns true when the initial configuration for this job is complete. - * - * @return bool - */ - public function configurationComplete(): bool - { - if ($this->importJob->stage === 'ready_to run') { - return true; - } - - return false; - } - - /** - * @param ImportJob $job - */ - public function setJob(ImportJob $job): void - { - $this->importJob = $job; - $this->repository = app(ImportJobRepositoryInterface::class); - $this->repository->setUser($job->user); - } } diff --git a/app/Support/Import/Configuration/File/ConfigurationInterface.php b/app/Support/Import/Configuration/File/ConfigurationInterface.php index dbbb74a446..e9344f35a8 100644 --- a/app/Support/Import/Configuration/File/ConfigurationInterface.php +++ b/app/Support/Import/Configuration/File/ConfigurationInterface.php @@ -30,6 +30,15 @@ use Illuminate\Support\MessageBag; */ interface ConfigurationInterface { + /** + * Store data associated with current stage. + * + * @param array $data + * + * @return MessageBag + */ + public function configureJob(array $data): MessageBag; + /** * Get the data necessary to show the configuration screen. * @@ -39,17 +48,6 @@ interface ConfigurationInterface /** * @param ImportJob $job - * - * @return ConfigurationInterface */ - public function setJob(ImportJob $job); - - /** - * Store data associated with current stage. - * - * @param array $data - * - * @return MessageBag - */ - public function configureJob(array $data): MessageBag; + public function setJob(ImportJob $job): void; } diff --git a/app/Support/Import/Configuration/File/ConfigureMappingHandler.php b/app/Support/Import/Configuration/File/ConfigureMappingHandler.php new file mode 100644 index 0000000000..b5f2b52599 --- /dev/null +++ b/app/Support/Import/Configuration/File/ConfigureMappingHandler.php @@ -0,0 +1,296 @@ +. + */ + +declare(strict_types=1); + +namespace FireflyIII\Support\Import\Configuration\File; + +use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Helpers\Attachments\AttachmentHelperInterface; +use FireflyIII\Import\Mapper\MapperInterface; +use FireflyIII\Models\Attachment; +use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use Illuminate\Support\Collection; +use Illuminate\Support\MessageBag; +use League\Csv\Exception; +use League\Csv\Reader; +use Log; + +/** + * Class ConfigureMappingHandler + */ +class ConfigureMappingHandler implements ConfigurationInterface +{ + /** @var AttachmentHelperInterface */ + private $attachments; + /** @var array */ + private $columnConfig; + /** @var ImportJob */ + private $importJob; + /** @var ImportJobRepositoryInterface */ + private $repository; + + /** + * Store data associated with current stage. + * + * @param array $data + * + * @return MessageBag + */ + public function configureJob(array $data): MessageBag + { + return new MessageBag; + } + + /** + * Get the data necessary to show the configuration screen. + * + * @return array + * @throws FireflyException + */ + public function getNextData(): array + { + $config = $this->importJob->configuration; + $columnConfig = $this->doColumnConfig($config); + + // in order to actually map we also need to read the FULL file. + try { + $reader = $this->getReader(); + } catch (Exception $e) { + Log::error($e->getMessage()); + throw new FireflyException('Cannot get reader: ' . $e->getMessage()); + } + // + // if ($config['has-headers']) { + // $offset = 1; + // } + // $stmt = (new Statement)->offset($offset); + // $results = $stmt->process($reader); + // $this->validSpecifics = array_keys(config('csv.import_specifics')); + // $indexes = array_keys($this->data); + // $rowIndex = 0; + // foreach ($results as $rowIndex => $row) { + // $row = $this->runSpecifics($row); + // + // //do something here + // foreach ($indexes as $index) { // this is simply 1, 2, 3, etc. + // if (!isset($row[$index])) { + // // don't really know how to handle this. Just skip, for now. + // continue; + // } + // $value = trim($row[$index]); + // if (\strlen($value) > 0) { + // // we can do some preprocessing here, + // // which is exclusively to fix the tags: + // if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) { + // /** @var PreProcessorInterface $preProcessor */ + // $preProcessor = app($this->data[$index]['preProcessMap']); + // $result = $preProcessor->run($value); + // $this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result); + // + // Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]); + // Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]); + // Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]); + // + // continue; + // } + // + // $this->data[$index]['values'][] = $value; + // } + // } + // } + // $setIndexes = array_keys($this->data); + // foreach ($setIndexes as $index) { + // $this->data[$index]['values'] = array_unique($this->data[$index]['values']); + // asort($this->data[$index]['values']); + // // if the count of this array is zero, there is nothing to map. + // if (\count($this->data[$index]['values']) === 0) { + // unset($this->data[$index]); + // } + // } + // unset($setIndexes); + // + // // save number of rows, thus number of steps, in job: + // $steps = $rowIndex * 5; + // $extended = $this->job->extended_status; + // $extended['steps'] = $steps; + // $this->job->extended_status = $extended; + // $this->job->save(); + // + // return $this->data; + // */ + } + + /** + * @param ImportJob $job + */ + public function setJob(ImportJob $job): void + { + $this->importJob = $job; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->repository->setUser($job->user); + $this->attachments = app(AttachmentHelperInterface::class); + $this->columnConfig = []; + } + + /** + * Create the "mapper" class that will eventually return the correct data for the user + * to map against. For example: a list of asset accounts. A list of budgets. A list of tags. + * + * @param string $column + * + * @return MapperInterface + * @throws FireflyException + */ + private function createMapper(string $column): MapperInterface + { + $mapperClass = config('csv.import_roles.' . $column . '.mapper'); + $mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass); + if (!class_exists($mapperName)) { + throw new FireflyException(sprintf('Class "%s" does not exist. Cannot map "%s"', $mapperName, $column)); + } + + return app($mapperName); + } + + /** + * For each column in the configuration of the job, will: + * - validate the role. + * - validate if it can be used for mapping + * - if so, create an entry in $columnConfig + * + * @param array $config + * + * @return array the column configuration. + * @throws FireflyException + */ + private function doColumnConfig(array $config): array + { + /** @var array $requestMapping */ + $requestMapping = $config['column-do-mapping'] ?? []; + $columnConfig = []; + /** + * @var int + * @var bool $mustBeMapped + */ + foreach ($requestMapping as $index => $requested) { + // sanitize column name, so we're sure it's valid. + $column = $this->sanitizeColumnName($config['column-roles'][$index] ?? '_ignore'); + $doMapping = $this->doMapOfColumn($column, $requested); + if ($doMapping) { + // user want to map this column. And this is possible. + $columnConfig[$index] = [ + 'name' => $column, + 'options' => $this->createMapper($column)->getMap(), + 'preProcessMap' => $this->getPreProcessorName($column), + 'values' => [], + ]; + } + } + + return $columnConfig; + } + + /** + * For each $name given, and if the user wants to map the column, will return + * true when the column can also be mapped. + * + * Unmappable columns will always return false. + * Mappable columns will return $requested. + * + * @param string $name + * @param bool $requested + * + * @return bool + */ + private function doMapOfColumn(string $name, bool $requested): bool + { + $canBeMapped = config('csv.import_roles.' . $name . '.mappable'); + + return $canBeMapped && $requested; + } + + /** + * Will return the name of the pre-processor: a special class that will clean up any input that may be found + * in the users input (aka the file uploaded). Only two examples exist at this time: a space or comma separated + * list of tags. + * + * @param string $column + * + * @return string + */ + private function getPreProcessorName(string $column): string + { + $name = ''; + $hasPreProcess = config(sprintf('csv.import_roles.%s.pre-process-map', $column)); + $preProcessClass = config(sprintf('csv.import_roles.%s.pre-process-mapper', $column)); + + if (null !== $hasPreProcess && true === $hasPreProcess && null !== $preProcessClass) { + $name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass); + } + + return $name; + } + + /** + * Return an instance of a CSV file reader so content of the file can be read. + * + * @throws \League\Csv\Exception + */ + private function getReader(): Reader + { + $content = ''; + /** @var Collection $collection */ + $collection = $this->importJob->attachments; + /** @var Attachment $attachment */ + foreach ($collection as $attachment) { + if ($attachment->filename === 'import_file') { + $content = $this->attachments->getAttachmentContent($attachment); + break; + } + } + $config = $this->repository->getConfiguration($this->importJob); + $reader = Reader::createFromString($content); + $reader->setDelimiter($config['delimiter']); + + return $reader; + } + + /** + * For each given column name, will return either the name (when it's a valid one) + * or return the _ignore column. + * + * @param string $name + * + * @return string + */ + private function sanitizeColumnName(string $name): string + { + /** @var array $validColumns */ + $validColumns = array_keys(config('csv.import_roles')); + if (!\in_array($name, $validColumns, true)) { + $name = '_ignore'; + } + + return $name; + } +} \ No newline at end of file diff --git a/app/Support/Import/Configuration/File/ConfigureRolesHandler.php b/app/Support/Import/Configuration/File/ConfigureRolesHandler.php new file mode 100644 index 0000000000..b24266cbf6 --- /dev/null +++ b/app/Support/Import/Configuration/File/ConfigureRolesHandler.php @@ -0,0 +1,394 @@ +. + */ + +declare(strict_types=1); + +namespace FireflyIII\Support\Import\Configuration\File; + +use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Helpers\Attachments\AttachmentHelperInterface; +use FireflyIII\Import\Specifics\SpecificInterface; +use FireflyIII\Models\Attachment; +use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use Illuminate\Support\Collection; +use Illuminate\Support\MessageBag; +use League\Csv\Exception; +use League\Csv\Reader; +use League\Csv\Statement; +use Log; + +/** + * Class ConfigureRolesHandler + */ +class ConfigureRolesHandler implements ConfigurationInterface +{ + /** @var AttachmentHelperInterface */ + private $attachments; + /** @var array */ + private $examples; + /** @var ImportJob */ + private $importJob; + /** @var ImportJobRepositoryInterface */ + private $repository; + /** @var int */ + private $totalColumns; + + /** + * Store data associated with current stage. + * + * @param array $data + * + * @return MessageBag + */ + public function configureJob(array $data): MessageBag + { + $config = $this->importJob->configuration; + $count = $config['column-count']; + for ($i = 0; $i < $count; ++$i) { + $role = $data['role'][$i] ?? '_ignore'; + $mapping = (isset($data['map'][$i]) && $data['map'][$i] === '1'); + $config['column-roles'][$i] = $role; + $config['column-do-mapping'][$i] = $mapping; + Log::debug(sprintf('Column %d has been given role %s (mapping: %s)', $i, $role, var_export($mapping, true))); + } + $config = $this->ignoreUnmappableColumns($config); + $messages = $this->configurationComplete($config); + + if ($messages->count() === 0) { + $this->repository->setStage($this->importJob, 'ready_to_run'); + if ($this->isMappingNecessary($config)) { + $this->repository->setStage($this->importJob, 'map'); + } + $this->repository->setConfiguration($this->importJob, $config); + } + + return $messages; + } + + /** + * Get the data necessary to show the configuration screen. + * + * @return array + * @throws FireflyException + */ + public function getNextData(): array + { + try { + $reader = $this->getReader(); + } catch (Exception $e) { + Log::error($e->getMessage()); + throw new FireflyException($e->getMessage()); + } + $headers = $this->getHeaders($reader); + + // get example rows: + $this->getExamples($reader); + + return [ + 'examples' => $this->examples, + 'roles' => $this->getRoles(), + 'total' => $this->totalColumns, + 'headers' => $headers, + ]; + } + + /** + * Set job and some start values. + * + * @param ImportJob $job + */ + public function setJob(ImportJob $job): void + { + $this->importJob = $job; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->repository->setUser($job->user); + $this->attachments = app(AttachmentHelperInterface::class); + $this->totalColumns = 0; + $this->examples = []; + } + + /** + * Verifies that the configuration of the job is actually complete, and valid. + * + * @param array $config + * + * @return MessageBag + */ + private function configurationComplete(array $config): MessageBag + { + $count = $config['column-count']; + $assigned = 0; + + // check if data actually contains amount column (foreign amount does not count) + $hasAmount = false; + $hasForeignAmount = false; + $hasForeignCode = false; + for ($i = 0; $i < $count; ++$i) { + $role = $config['column-roles'][$i] ?? '_ignore'; + if ('_ignore' !== $role) { + ++$assigned; + } + if (\in_array($role, ['amount', 'amount_credit', 'amount_debit'])) { + $hasAmount = true; + } + if ($role === 'foreign-currency-code') { + $hasForeignCode = true; + } + if ($role === 'amount_foreign') { + $hasForeignAmount = true; + } + } + + // all assigned and correct foreign info + if ($assigned > 0 && $hasAmount && ($hasForeignCode === $hasForeignAmount)) { + return new MessageBag; + } + if (0 === $assigned || !$hasAmount) { + $message = (string)trans('import.job_config_roles_rwarning'); + $messages = new MessageBag(); + $messages->add('error', $message); + + return $messages; + } + + // warn if has foreign amount but no currency code: + if ($hasForeignAmount && !$hasForeignCode) { + $message = (string)trans('import.job_config_roles_fa_warning'); + $messages = new MessageBag(); + $messages->add('error', $message); + + return $messages; + } + + + + return new MessageBag; + } + + /** + * Extracts example data from a single row and store it in the class. + * + * @param array $row + */ + private function getExampleFromRow(array $row): void + { + foreach ($row as $index => $value) { + $value = trim($value); + if (\strlen($value) > 0) { + $this->examples[$index][] = $value; + } + } + } + + /** + * Return a bunch of examples from the CSV file the user has uploaded. + * + * @param Reader $reader + * + * @throws FireflyException + */ + private function getExamples(Reader $reader): void + { + // configure example data: + $config = $this->importJob->configuration; + $limit = (int)config('csv.example_rows', 5); + $offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0; + + // make statement. + try { + $stmt = (new Statement)->limit($limit)->offset($offset); + } catch (Exception $e) { + Log::error($e->getMessage()); + throw new FireflyException($e->getMessage()); + } + + // grab the records: + $records = $stmt->process($reader); + /** @var array $row */ + foreach ($records as $row) { + $row = array_values($row); + $row = $this->processSpecifics($row); + $count = \count($row); + $this->totalColumns = $count > $this->totalColumns ? $count : $this->totalColumns; + $this->getExampleFromRow($row); + } + // save column count: + $this->saveColumCount(); + $this->makeExamplesUnique(); + } + + /** + * Get the header row, if one is present. + * + * @param Reader $reader + * + * @return array + * @throws FireflyException + */ + private function getHeaders(Reader $reader): array + { + $headers = []; + $config = $this->importJob->configuration; + if ($config['has-headers']) { + try { + $stmt = (new Statement)->limit(1)->offset(0); + $records = $stmt->process($reader); + $headers = $records->fetchOne(0); + } catch (Exception $e) { + Log::error($e->getMessage()); + throw new FireflyException($e->getMessage()); + } + Log::debug('Detected file headers:', $headers); + } + + return $headers; + } + + /** + * Return an instance of a CSV file reader so content of the file can be read. + * + * @throws \League\Csv\Exception + */ + private function getReader(): Reader + { + $content = ''; + /** @var Collection $collection */ + $collection = $this->importJob->attachments; + /** @var Attachment $attachment */ + foreach ($collection as $attachment) { + if ($attachment->filename === 'import_file') { + $content = $this->attachments->getAttachmentContent($attachment); + break; + } + } + $config = $this->repository->getConfiguration($this->importJob); + $reader = Reader::createFromString($content); + $reader->setDelimiter($config['delimiter']); + + return $reader; + } + + /** + * Returns all possible roles and translate their name. Then sort them. + * + * @return array + */ + private function getRoles(): array + { + $roles = []; + foreach (array_keys(config('csv.import_roles')) as $role) { + $roles[$role] = trans('import.column_' . $role); + } + asort($roles); + + return $roles; + } + + /** + * If the user has checked columns that cannot be mapped to any value, this function will + * uncheck them and return the configuration again. + * + * @param array $config + * + * @return array + */ + private function ignoreUnmappableColumns(array $config): array + { + $count = $config['column-count']; + for ($i = 0; $i < $count; ++$i) { + $role = $config['column-roles'][$i] ?? '_ignore'; + $mapping = $config['column-do-mapping'][$i] ?? false; + // if the column can be mapped depends on the config: + $canMap = (bool)config(sprintf('csv.import_roles.%s.mappable', $role)); + $mapping = $mapping && $canMap; + $config['column-do-mapping'][$i] = $mapping; + } + + return $config; + } + + /** + * Returns false when it's not necessary to map values. This saves time and is user friendly + * (will skip to the next screen). + * + * @param array $config + * + * @return bool + */ + private function isMappingNecessary(array $config): bool + { + $count = $config['column-count']; + $toBeMapped = 0; + for ($i = 0; $i < $count; ++$i) { + $mapping = $config['column-do-mapping'][$i] ?? false; + if (true === $mapping) { + ++$toBeMapped; + } + } + + return !(0 === $toBeMapped); + } + + /** + * Make sure that the examples do not contain double data values. + */ + private function makeExamplesUnique(): void + { + foreach ($this->examples as $index => $values) { + $this->examples[$index] = array_unique($values); + } + } + + /** + * if the user has configured specific fixes to be applied, they must be applied to the example data as well. + * + * @param array $row + * + * @return array + */ + private function processSpecifics(array $row): array + { + $config = $this->importJob->configuration; + $specifics = $config['specifics'] ?? []; + $names = array_keys($specifics); + foreach ($names as $name) { + /** @var SpecificInterface $specific */ + $specific = app('FireflyIII\Import\Specifics\\' . $name); + $row = $specific->run($row); + } + + return $row; + + } + + /** + * Save the column count in the job. It's used in a later stage. + * + * @return void + */ + private function saveColumCount(): void + { + $config = $this->importJob->configuration; + $config['column-count'] = $this->totalColumns; + $this->repository->setConfiguration($this->importJob, $config); + } +} \ No newline at end of file diff --git a/app/Support/Import/Configuration/File/ConfigureUploadHandler.php b/app/Support/Import/Configuration/File/ConfigureUploadHandler.php index aedc183a41..7fd36b8204 100644 --- a/app/Support/Import/Configuration/File/ConfigureUploadHandler.php +++ b/app/Support/Import/Configuration/File/ConfigureUploadHandler.php @@ -83,10 +83,8 @@ class ConfigureUploadHandler implements ConfigurationInterface /** * @param ImportJob $job - * - * @return ConfigurationInterface */ - public function setJob(ImportJob $job) + public function setJob(ImportJob $job): void { $this->importJob = $job; $this->repository = app(ImportJobRepositoryInterface::class); diff --git a/resources/lang/en_US/import.php b/resources/lang/en_US/import.php index dfcc850526..f60d78ef93 100644 --- a/resources/lang/en_US/import.php +++ b/resources/lang/en_US/import.php @@ -98,7 +98,7 @@ return [ 'job_config_uc_apply_rules_title' => 'Apply rules', 'job_config_uc_apply_rules_text' => 'Applies your rules to every imported transaction. Note that this slows the import significantly.', 'job_config_uc_specifics_title' => 'Bank-specific options', - 'job_config_uc_specifics_txt' => 'Some banks deliver badly formatted files. Firefly III can fix those automatically. If your bank delivers such files, open an issue on GitHub.', + 'job_config_uc_specifics_txt' => 'Some banks deliver badly formatted files. Firefly III can fix those automatically. If your bank delivers such files but it\'s not listed here, please open an issue on GitHub.', 'job_config_uc_submit' => 'Continue', 'invalid_import_account' => 'You have selected an invalid account to import into.', // specifics: @@ -112,6 +112,18 @@ return [ 'specific_rabo_descr' => 'Fixes potential problems with Rabobank files', 'specific_pres_name' => 'President\'s Choice Financial CA', 'specific_pres_descr' => 'Fixes potential problems with PC files', + // job configuration for file provider (stage: roles) + 'job_config_roles_title' => 'Import setup (3/4) - Define each column\'s role', + 'job_config_roles_text' => 'Each column in your CSV file contains certain data. Please indicate what kind of data the importer should expect. The option to "map" data means that you will link each entry found in the column to a value in your database. An often mapped column is the column that contains the IBAN of the opposing account. That can be easily matched to IBAN\'s present in your database already.', + 'job_config_roles_submit' => 'Continue', + 'job_config_roles_column_name' => 'Name of column', + 'job_config_roles_column_example' => 'Column example data', + 'job_config_roles_column_role' => 'Column data meaning', + 'job_config_roles_do_map_value' => 'Map these values', + 'job_config_roles_no_example' => 'No example data available', + 'job_config_roles_fa_warning' => 'If you mark a column as containing an amount in a foreign currency, you must also set the column that contains which currency it is.', + 'job_config_roles_rwarning' => 'At the very least, mark one column as the amount-column. It is advisable to also select a column for the description, date and the opposing account.', + 'job_config_roles_colum_count' => 'Column', // import status page: @@ -137,6 +149,55 @@ return [ // general errors and warnings: 'bad_job_status' => 'To access this page, your import job cannot have status ":status".', + // column roles for CSV import: + 'column__ignore' => '(ignore this column)', + 'column_account-iban' => 'Asset account (IBAN)', + 'column_account-id' => 'Asset account ID (matching FF3)', + 'column_account-name' => 'Asset account (name)', + 'column_amount' => 'Amount', + 'column_amount_foreign' => 'Amount (in foreign currency)', + 'column_amount_debit' => 'Amount (debit column)', + 'column_amount_credit' => 'Amount (credit column)', + 'column_amount-comma-separated' => 'Amount (comma as decimal separator)', + 'column_bill-id' => 'Bill ID (matching FF3)', + 'column_bill-name' => 'Bill name', + 'column_budget-id' => 'Budget ID (matching FF3)', + 'column_budget-name' => 'Budget name', + 'column_category-id' => 'Category ID (matching FF3)', + 'column_category-name' => 'Category name', + 'column_currency-code' => 'Currency code (ISO 4217)', + 'column_foreign-currency-code' => 'Foreign currency code (ISO 4217)', + 'column_currency-id' => 'Currency ID (matching FF3)', + 'column_currency-name' => 'Currency name (matching FF3)', + 'column_currency-symbol' => 'Currency symbol (matching FF3)', + 'column_date-interest' => 'Interest calculation date', + 'column_date-book' => 'Transaction booking date', + 'column_date-process' => 'Transaction process date', + 'column_date-transaction' => 'Date', + 'column_date-due' => 'Transaction due date', + 'column_date-payment' => 'Transaction payment date', + 'column_date-invoice' => 'Transaction invoice date', + 'column_description' => 'Description', + 'column_opposing-iban' => 'Opposing account (IBAN)', + 'column_opposing-bic' => 'Opposing account (BIC)', + 'column_opposing-id' => 'Opposing account ID (matching FF3)', + 'column_external-id' => 'External ID', + 'column_opposing-name' => 'Opposing account (name)', + 'column_rabo-debit-credit' => 'Rabobank specific debit/credit indicator', + 'column_ing-debit-credit' => 'ING specific debit/credit indicator', + 'column_sepa-ct-id' => 'SEPA end-to-end Identifier', + 'column_sepa-ct-op' => 'SEPA Opposing Account Identifier', + 'column_sepa-db' => 'SEPA Mandate Identifier', + 'column_sepa-cc' => 'SEPA Clearing Code', + 'column_sepa-ci' => 'SEPA Creditor Identifier', + 'column_sepa-ep' => 'SEPA External Purpose', + 'column_sepa-country' => 'SEPA Country Code', + 'column_tags-comma' => 'Tags (comma separated)', + 'column_tags-space' => 'Tags (space separated)', + 'column_account-number' => 'Asset account (account number)', + 'column_opposing-number' => 'Opposing account (account number)', + 'column_note' => 'Note(s)', + 'column_internal-reference' => 'Internal reference', // status of import: // 'status_wait_title' => 'Please hold...', diff --git a/resources/views/import/file/map.twig b/resources/views/import/file/map.twig index 5ae646b42c..d765d54d47 100644 --- a/resources/views/import/file/map.twig +++ b/resources/views/import/file/map.twig @@ -1,7 +1,7 @@ {% extends "./layout/default" %} {% block breadcrumbs %} - {{ Breadcrumbs.render(Route.getCurrentRoute.getName, job) }} + {{ Breadcrumbs.render(Route.getCurrentRoute.getName, importJob) }} {% endblock %} {% block content %} @@ -27,7 +27,7 @@ -
+ diff --git a/resources/views/import/file/roles.twig b/resources/views/import/file/roles.twig index 8730ccaf0f..a983dc1b35 100644 --- a/resources/views/import/file/roles.twig +++ b/resources/views/import/file/roles.twig @@ -1,7 +1,7 @@ {% extends "./layout/default" %} {% block breadcrumbs %} - {{ Breadcrumbs.render(Route.getCurrentRoute.getName, job) }} + {{ Breadcrumbs.render(Route.getCurrentRoute.getName, importJob) }} {% endblock %} {% block content %} @@ -10,36 +10,34 @@
-

{{ trans('import.csv_roles_title') }}

+

{{ trans('import.job_config_roles_title') }}

- {{ trans('import.csv_roles_text') }} + {{ trans('import.job_config_roles_text') }}

- + -
-

{{ trans('import.csv_roles_table') }}

+

{{ trans('import.job_config_input') }}

- - - - - + + + + {% for i in 0..(data.total -1) %} @@ -47,14 +45,14 @@ @@ -80,7 +78,6 @@
{{ trans('import.csv_roles_column_name') }}{{ trans('import.csv_roles_column_example') }}{{ trans('import.csv_roles_column_role') }}{{ trans('import.csv_roles_do_map_value') }}{{ trans('import.job_config_roles_column_name') }}{{ trans('import.job_config_roles_column_example') }}{{ trans('import.job_config_roles_column_role') }}{{ trans('import.job_config_roles_do_map_value') }}
{% if data.headers[i] == '' %} - {{ trans('import.csv_roles_column') }} #{{ loop.index }} + {{ trans('import.job_config_roles_colum_count') }} #{{ loop.index }} {% else %} {{ data.headers[i] }} {% endif %} {% if data.examples[i]|length == 0 %} - {{ trans('import.csv_roles_no_example_data') }} + {{ trans('import.job_config_roles_no_example') }} {% else %} {% for example in data.examples[i] %} {{ example }}
@@ -64,12 +62,12 @@ 		{{ Form.select(('role['~loop.index0~']'), data.roles, - job.configuration['column-roles'][loop.index0], + importJob.configuration['column-roles'][loop.index0], {class: 'form-control'}) }} {{ Form.checkbox(('map['~loop.index0~']'),1, - job.configuration['column-do-mapping'][loop.index0] + importJob.configuration['column-do-mapping'][loop.index0] ) }}
-
@@ -91,7 +88,7 @@