mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-02-25 18:45:27 -06:00
Start processing files.
This commit is contained in:
James Cole
parent
1209f3b39a
commit
690c9203c8
@ -58,11 +58,10 @@ class FileRoutine implements RoutineInterface
|
||||
case 'ready_to_run':
|
||||
// get processor, depending on file type
|
||||
// is just CSV for now.
|
||||
|
||||
$processor = $this->getProcessor();
|
||||
$processor->setJob($this->importJob);
|
||||
$transactions = $processor->run();
|
||||
|
||||
|
||||
// make processor run.
|
||||
// then done!
|
||||
// move to status 'processor_finished'.
|
||||
|
||||
@ -1,144 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* Initial.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\File;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @codeCoverageIgnore
|
||||
* Class Initial.
|
||||
*/
|
||||
class Initial
|
||||
{
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
|
||||
/** @var string */
|
||||
private $warning = '';
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
Log::debug('Constructed Initial.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
$importFileTypes = [];
|
||||
$defaultImportType = config('import.options.file.default_import_format');
|
||||
|
||||
foreach (config('import.options.file.import_formats') as $type) {
|
||||
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
|
||||
}
|
||||
|
||||
return [
|
||||
'default_type' => $defaultImportType,
|
||||
'file_types' => $importFileTypes,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return possible warning to user.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getWarningMessage(): string
|
||||
{
|
||||
return $this->warning;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
* @return ConfigurationInterface
|
||||
*/
|
||||
public function setJob(ImportJob $job): ConfigurationInterface
|
||||
{
|
||||
$this->job = $job;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->repository->setUser($job->user);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
Log::debug('Now in storeConfiguration for file Upload.');
|
||||
$config = $this->getConfig();
|
||||
$type = $data['import_file_type'] ?? 'csv'; // assume it's a CSV file.
|
||||
$config['file-type'] = \in_array($type, config('import.options.file.import_formats'), true) ? $type : 'csv';
|
||||
|
||||
// update config:
|
||||
$this->repository->setConfiguration($this->job, $config);
|
||||
|
||||
// make repository process file:
|
||||
$uploaded = $this->repository->processFile($this->job, $data['import_file'] ?? null);
|
||||
Log::debug(sprintf('Result of upload is %s', var_export($uploaded, true)));
|
||||
|
||||
// process config, if present:
|
||||
if (isset($data['configuration_file'])) {
|
||||
Log::debug('Will also upload configuration.');
|
||||
$this->repository->processConfiguration($this->job, $data['configuration_file']);
|
||||
}
|
||||
|
||||
if (false === $uploaded) {
|
||||
$this->warning = (string)trans('firefly.upload_error');
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// if file was upload safely, assume we can go to the next stage:
|
||||
$config = $this->getConfig();
|
||||
$config['stage'] = 'upload-config';
|
||||
$this->repository->setConfiguration($this->job, $config);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Short hand method.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function getConfig(): array
|
||||
{
|
||||
return $this->repository->getConfiguration($this->job);
|
||||
}
|
||||
}
|
||||
@ -1,319 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* Map.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\File;
|
||||
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Import\Mapper\MapperInterface;
|
||||
use FireflyIII\Import\MapperPreProcess\PreProcessorInterface;
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use League\Csv\Reader;
|
||||
use League\Csv\Statement;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @codeCoverageIgnore
|
||||
* Class Mapping.
|
||||
*/
|
||||
class Map
|
||||
{
|
||||
/** @var array that holds each column to be mapped by the user */
|
||||
private $data = [];
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
/** @var array */
|
||||
private $validSpecifics = [];
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*
|
||||
* @throws FireflyException
|
||||
* @throws \League\Csv\Exception
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
$this->getMappableColumns();
|
||||
|
||||
// in order to actually map we also need all possible values from the CSV file.
|
||||
$config = $this->getConfig();
|
||||
$content = $this->repository->uploadFileContents($this->job);
|
||||
$offset = 0;
|
||||
/** @var Reader $reader */
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
if ($config['has-headers']) {
|
||||
$offset = 1;
|
||||
}
|
||||
$stmt = (new Statement)->offset($offset);
|
||||
$results = $stmt->process($reader);
|
||||
$this->validSpecifics = array_keys(config('csv.import_specifics'));
|
||||
$indexes = array_keys($this->data);
|
||||
$rowIndex = 0;
|
||||
foreach ($results as $rowIndex => $row) {
|
||||
$row = $this->runSpecifics($row);
|
||||
|
||||
//do something here
|
||||
foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
|
||||
if (!isset($row[$index])) {
|
||||
// don't really know how to handle this. Just skip, for now.
|
||||
continue;
|
||||
}
|
||||
$value = trim($row[$index]);
|
||||
if (\strlen($value) > 0) {
|
||||
// we can do some preprocessing here,
|
||||
// which is exclusively to fix the tags:
|
||||
if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) {
|
||||
/** @var PreProcessorInterface $preProcessor */
|
||||
$preProcessor = app($this->data[$index]['preProcessMap']);
|
||||
$result = $preProcessor->run($value);
|
||||
$this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result);
|
||||
|
||||
Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
|
||||
Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
|
||||
Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->data[$index]['values'][] = $value;
|
||||
}
|
||||
}
|
||||
}
|
||||
$setIndexes = array_keys($this->data);
|
||||
foreach ($setIndexes as $index) {
|
||||
$this->data[$index]['values'] = array_unique($this->data[$index]['values']);
|
||||
asort($this->data[$index]['values']);
|
||||
// if the count of this array is zero, there is nothing to map.
|
||||
if (\count($this->data[$index]['values']) === 0) {
|
||||
unset($this->data[$index]);
|
||||
}
|
||||
}
|
||||
unset($setIndexes);
|
||||
|
||||
// save number of rows, thus number of steps, in job:
|
||||
$steps = $rowIndex * 5;
|
||||
$extended = $this->job->extended_status;
|
||||
$extended['steps'] = $steps;
|
||||
$this->job->extended_status = $extended;
|
||||
$this->job->save();
|
||||
|
||||
return $this->data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return possible warning to user.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getWarningMessage(): string
|
||||
{
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
* @return ConfigurationInterface
|
||||
*/
|
||||
public function setJob(ImportJob $job): ConfigurationInterface
|
||||
{
|
||||
$this->job = $job;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->repository->setUser($job->user);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
$config = $this->getConfig();
|
||||
|
||||
if (isset($data['mapping'])) {
|
||||
foreach ($data['mapping'] as $index => $array) {
|
||||
$config['column-mapping-config'][$index] = [];
|
||||
foreach ($array as $value => $mapId) {
|
||||
$mapId = (int)$mapId;
|
||||
if (0 !== $mapId) {
|
||||
$config['column-mapping-config'][$index][$value] = $mapId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set thing to be completed.
|
||||
$config['stage'] = 'ready';
|
||||
$this->saveConfig($config);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
*
|
||||
* @return MapperInterface
|
||||
*/
|
||||
private function createMapper(string $column): MapperInterface
|
||||
{
|
||||
$mapperClass = config('csv.import_roles.' . $column . '.mapper');
|
||||
$mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass);
|
||||
|
||||
return app($mapperName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Short hand method.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function getConfig(): array
|
||||
{
|
||||
return $this->repository->getConfiguration($this->job);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function getMappableColumns(): bool
|
||||
{
|
||||
$config = $this->getConfig();
|
||||
/**
|
||||
* @var int
|
||||
* @var bool $mustBeMapped
|
||||
*/
|
||||
foreach ($config['column-do-mapping'] as $index => $mustBeMapped) {
|
||||
$column = $this->validateColumnName($config['column-roles'][$index] ?? '_ignore');
|
||||
$shouldMap = $this->shouldMapColumn($column, $mustBeMapped);
|
||||
if ($shouldMap) {
|
||||
// create configuration entry for this specific column and add column to $this->data array for later processing.
|
||||
$this->data[$index] = [
|
||||
'name' => $column,
|
||||
'index' => $index,
|
||||
'options' => $this->createMapper($column)->getMap(),
|
||||
'preProcessMap' => $this->getPreProcessorName($column),
|
||||
'values' => [],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
private function getPreProcessorName(string $column): string
|
||||
{
|
||||
$name = '';
|
||||
$hasPreProcess = config('csv.import_roles.' . $column . '.pre-process-map');
|
||||
$preProcessClass = config('csv.import_roles.' . $column . '.pre-process-mapper');
|
||||
|
||||
if (null !== $hasPreProcess && true === $hasPreProcess && null !== $preProcessClass) {
|
||||
$name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
|
||||
}
|
||||
|
||||
return $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $row
|
||||
*
|
||||
* @return array
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function runSpecifics(array $row): array
|
||||
{
|
||||
// run specifics here:
|
||||
// and this is the point where the specifix go to work.
|
||||
$config = $this->getConfig();
|
||||
$specifics = $config['specifics'] ?? [];
|
||||
$names = array_keys($specifics);
|
||||
foreach ($names as $name) {
|
||||
if (!\in_array($name, $this->validSpecifics)) {
|
||||
throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
|
||||
}
|
||||
$class = config('csv.import_specifics.' . $name);
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app($class);
|
||||
|
||||
// it returns the row, possibly modified:
|
||||
$row = $specific->run($row);
|
||||
}
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $array
|
||||
*/
|
||||
private function saveConfig(array $array)
|
||||
{
|
||||
$this->repository->setConfiguration($this->job, $array);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
* @param bool $mustBeMapped
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
private function shouldMapColumn(string $column, bool $mustBeMapped): bool
|
||||
{
|
||||
$canBeMapped = config('csv.import_roles.' . $column . '.mappable');
|
||||
|
||||
return $canBeMapped && $mustBeMapped;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
*
|
||||
* @return string
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function validateColumnName(string $column): string
|
||||
{
|
||||
// is valid column?
|
||||
$validColumns = array_keys(config('csv.import_roles'));
|
||||
if (!\in_array($column, $validColumns)) {
|
||||
throw new FireflyException(sprintf('"%s" is not a valid column.', $column));
|
||||
}
|
||||
|
||||
return $column;
|
||||
}
|
||||
}
|
||||
@ -1,367 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* Roles.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\File;
|
||||
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use League\Csv\Reader;
|
||||
use League\Csv\Statement;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @codeCoverageIgnore
|
||||
* Class Roles.
|
||||
*/
|
||||
class Roles
|
||||
{
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
private $data = [];
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
/** @var string */
|
||||
private $warning = '';
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
*
|
||||
* @throws \League\Csv\Exception
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
$content = $this->repository->uploadFileContents($this->job);
|
||||
$config = $this->getConfig();
|
||||
$headers = [];
|
||||
$offset = 0;
|
||||
|
||||
// create CSV reader.
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
|
||||
// CSV headers. Ignore reader. Simply get the first row.
|
||||
if ($config['has-headers']) {
|
||||
$offset = 1;
|
||||
$stmt = (new Statement)->limit(1)->offset(0);
|
||||
$records = $stmt->process($reader);
|
||||
$headers = $records->fetchOne(0);
|
||||
Log::debug('Detected file headers:', $headers);
|
||||
}
|
||||
|
||||
// example rows:
|
||||
$stmt = (new Statement)->limit((int)config('csv.example_rows', 5))->offset($offset);
|
||||
// set data:
|
||||
$roles = $this->getRoles();
|
||||
asort($roles);
|
||||
$this->data = [
|
||||
'examples' => [],
|
||||
'roles' => $roles,
|
||||
'total' => 0,
|
||||
'headers' => $headers,
|
||||
];
|
||||
|
||||
$records = $stmt->process($reader);
|
||||
foreach ($records as $row) {
|
||||
$row = array_values($row);
|
||||
$row = $this->processSpecifics($row);
|
||||
$count = \count($row);
|
||||
$this->data['total'] = $count > $this->data['total'] ? $count : $this->data['total'];
|
||||
$this->processRow($row);
|
||||
}
|
||||
|
||||
$this->updateColumCount();
|
||||
$this->makeExamplesUnique();
|
||||
|
||||
return $this->data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return possible warning to user.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getWarningMessage(): string
|
||||
{
|
||||
return $this->warning;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
* @return ConfigurationInterface
|
||||
*/
|
||||
public function setJob(ImportJob $job): ConfigurationInterface
|
||||
{
|
||||
$this->job = $job;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->repository->setUser($job->user);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
Log::debug('Now in storeConfiguration of Roles.');
|
||||
$config = $this->getConfig();
|
||||
$count = $config['column-count'];
|
||||
for ($i = 0; $i < $count; ++$i) {
|
||||
$role = $data['role'][$i] ?? '_ignore';
|
||||
$mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false;
|
||||
$config['column-roles'][$i] = $role;
|
||||
$config['column-do-mapping'][$i] = $mapping;
|
||||
Log::debug(sprintf('Column %d has been given role %s (mapping: %s)', $i, $role, var_export($mapping, true)));
|
||||
}
|
||||
|
||||
$this->saveConfig($config);
|
||||
$this->ignoreUnmappableColumns();
|
||||
$res = $this->isRolesComplete();
|
||||
if ($res === true) {
|
||||
$config = $this->getConfig();
|
||||
$config['stage'] = 'map';
|
||||
$this->saveConfig($config);
|
||||
$this->isMappingNecessary();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Short hand method.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function getConfig(): array
|
||||
{
|
||||
return $this->repository->getConfiguration($this->job);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
private function getRoles(): array
|
||||
{
|
||||
$roles = [];
|
||||
foreach (array_keys(config('csv.import_roles')) as $role) {
|
||||
$roles[$role] = trans('import.column_' . $role);
|
||||
}
|
||||
|
||||
return $roles;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function ignoreUnmappableColumns(): bool
|
||||
{
|
||||
Log::debug('Now in ignoreUnmappableColumns()');
|
||||
$config = $this->getConfig();
|
||||
$count = $config['column-count'];
|
||||
for ($i = 0; $i < $count; ++$i) {
|
||||
$role = $config['column-roles'][$i] ?? '_ignore';
|
||||
$mapping = $config['column-do-mapping'][$i] ?? false;
|
||||
Log::debug(sprintf('Role for column %d is %s, and mapping is %s', $i, $role, var_export($mapping, true)));
|
||||
|
||||
if ('_ignore' === $role && true === $mapping) {
|
||||
$mapping = false;
|
||||
Log::debug(sprintf('Column %d has type %s so it cannot be mapped.', $i, $role));
|
||||
}
|
||||
$config['column-do-mapping'][$i] = $mapping;
|
||||
}
|
||||
$this->saveConfig($config);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function isMappingNecessary()
|
||||
{
|
||||
$config = $this->getConfig();
|
||||
$count = $config['column-count'];
|
||||
$toBeMapped = 0;
|
||||
for ($i = 0; $i < $count; ++$i) {
|
||||
$mapping = $config['column-do-mapping'][$i] ?? false;
|
||||
if (true === $mapping) {
|
||||
++$toBeMapped;
|
||||
}
|
||||
}
|
||||
Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped));
|
||||
if (0 === $toBeMapped) {
|
||||
$config['stage'] = 'ready';
|
||||
}
|
||||
|
||||
$this->saveConfig($config);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function isRolesComplete(): bool
|
||||
{
|
||||
$config = $this->getConfig();
|
||||
$count = $config['column-count'];
|
||||
$assigned = 0;
|
||||
|
||||
// check if data actually contains amount column (foreign amount does not count)
|
||||
$hasAmount = false;
|
||||
$hasForeignAmount = false;
|
||||
$hasForeignCode = false;
|
||||
for ($i = 0; $i < $count; ++$i) {
|
||||
$role = $config['column-roles'][$i] ?? '_ignore';
|
||||
if ('_ignore' !== $role) {
|
||||
++$assigned;
|
||||
}
|
||||
if (\in_array($role, ['amount', 'amount_credit', 'amount_debit'])) {
|
||||
$hasAmount = true;
|
||||
}
|
||||
if ($role === 'foreign-currency-code') {
|
||||
$hasForeignCode = true;
|
||||
}
|
||||
if ($role === 'amount_foreign') {
|
||||
$hasForeignAmount = true;
|
||||
}
|
||||
}
|
||||
Log::debug(
|
||||
sprintf(
|
||||
'Assigned is %d, hasAmount %s, hasForeignCode %s, hasForeignAmount %s',
|
||||
$assigned,
|
||||
var_export($hasAmount, true),
|
||||
var_export($hasForeignCode, true),
|
||||
var_export($hasForeignAmount, true)
|
||||
)
|
||||
);
|
||||
// all assigned and correct foreign info
|
||||
if ($assigned > 0 && $hasAmount && ($hasForeignCode === $hasForeignAmount)) {
|
||||
$this->warning = '';
|
||||
$this->saveConfig($config);
|
||||
Log::debug('isRolesComplete() returns true.');
|
||||
|
||||
return true;
|
||||
}
|
||||
// warn if has foreign amount but no currency code:
|
||||
if ($hasForeignAmount && !$hasForeignCode) {
|
||||
$this->warning = (string)trans('import.foreign_amount_warning');
|
||||
Log::debug('isRolesComplete() returns FALSE because foreign amount present without foreign code.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (0 === $assigned || !$hasAmount) {
|
||||
$this->warning = (string)trans('import.roles_warning');
|
||||
Log::debug('isRolesComplete() returns FALSE because no amount present.');
|
||||
|
||||
return false;
|
||||
}
|
||||
Log::debug('isRolesComplete() returns FALSE because no reason.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* make unique example data.
|
||||
*/
|
||||
private function makeExamplesUnique(): bool
|
||||
{
|
||||
foreach ($this->data['examples'] as $index => $values) {
|
||||
$this->data['examples'][$index] = array_unique($values);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $row
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
private function processRow(array $row): bool
|
||||
{
|
||||
foreach ($row as $index => $value) {
|
||||
$value = trim($value);
|
||||
if (\strlen($value) > 0) {
|
||||
$this->data['examples'][$index][] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* run specifics here:
|
||||
* and this is the point where the specifix go to work.
|
||||
*
|
||||
* @param array $row
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function processSpecifics(array $row): array
|
||||
{
|
||||
$config = $this->getConfig();
|
||||
$specifics = $config['specifics'] ?? [];
|
||||
$names = array_keys($specifics);
|
||||
foreach ($names as $name) {
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app('FireflyIII\Import\Specifics\\' . $name);
|
||||
$row = $specific->run($row);
|
||||
}
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $array
|
||||
*/
|
||||
private function saveConfig(array $array)
|
||||
{
|
||||
$this->repository->setConfiguration($this->job, $array);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function updateColumCount(): bool
|
||||
{
|
||||
$config = $this->getConfig();
|
||||
$count = $this->data['total'];
|
||||
$config['column-count'] = $count;
|
||||
$this->saveConfig($config);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,190 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* UploadConfig.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\File;
|
||||
|
||||
use FireflyIII\Models\AccountType;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @codeCoverageIgnore
|
||||
* Class UploadConfig.
|
||||
*/
|
||||
class UploadConfig
|
||||
{
|
||||
/** @var AccountRepositoryInterface */
|
||||
private $accountRepository;
|
||||
/**
|
||||
* @var ImportJob
|
||||
*/
|
||||
private $job;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
$accounts = $this->accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
|
||||
$delimiters = [
|
||||
',' => trans('form.csv_comma'),
|
||||
';' => trans('form.csv_semicolon'),
|
||||
'tab' => trans('form.csv_tab'),
|
||||
];
|
||||
$config = $this->getConfig();
|
||||
$config['date-format'] = $config['date-format'] ?? 'Ymd';
|
||||
$specifics = [];
|
||||
$this->saveConfig($config);
|
||||
|
||||
// collect specifics.
|
||||
foreach (config('csv.import_specifics') as $name => $className) {
|
||||
$specifics[$name] = [
|
||||
'name' => $className::getName(),
|
||||
'description' => $className::getDescription(),
|
||||
];
|
||||
}
|
||||
|
||||
$data = [
|
||||
'accounts' => app('expandedform')->makeSelectList($accounts),
|
||||
'specifix' => [],
|
||||
'delimiters' => $delimiters,
|
||||
'specifics' => $specifics,
|
||||
];
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return possible warning to user.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getWarningMessage(): string
|
||||
{
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
* @return ConfigurationInterface
|
||||
*/
|
||||
public function setJob(ImportJob $job): ConfigurationInterface
|
||||
{
|
||||
$this->job = $job;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->accountRepository = app(AccountRepositoryInterface::class);
|
||||
$this->repository->setUser($job->user);
|
||||
$this->accountRepository->setUser($job->user);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
Log::debug('Now in Initial::storeConfiguration()');
|
||||
$config = $this->getConfig();
|
||||
$importId = (int)($data['csv_import_account'] ?? 0.0);
|
||||
$account = $this->accountRepository->find($importId);
|
||||
$delimiter = (string)$data['csv_delimiter'];
|
||||
|
||||
// set "headers":
|
||||
$config['has-headers'] = (int)($data['has_headers'] ?? 0.0) === 1;
|
||||
$config['date-format'] = (string)$data['date_format'];
|
||||
$config['delimiter'] = 'tab' === $delimiter ? "\t" : $delimiter;
|
||||
$config['apply-rules'] = (int)($data['apply_rules'] ?? 0.0) === 1;
|
||||
|
||||
Log::debug('Entered import account.', ['id' => $importId]);
|
||||
|
||||
|
||||
if (null !== $account->id) {
|
||||
Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]);
|
||||
$config['import-account'] = $account->id;
|
||||
}
|
||||
|
||||
if (null === $account->id) {
|
||||
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
|
||||
}
|
||||
$config = $this->storeSpecifics($data, $config);
|
||||
Log::debug('Final config is ', $config);
|
||||
|
||||
// onto the next stage!
|
||||
|
||||
$config['stage'] = 'roles';
|
||||
$this->saveConfig($config);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Short hand method.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function getConfig(): array
|
||||
{
|
||||
return $this->repository->getConfiguration($this->job);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $array
|
||||
*/
|
||||
private function saveConfig(array $array)
|
||||
{
|
||||
$this->repository->setConfiguration($this->job, $array);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $data
|
||||
* @param array $config
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function storeSpecifics(array $data, array $config): array
|
||||
{
|
||||
// loop specifics.
|
||||
if (isset($data['specifics']) && \is_array($data['specifics'])) {
|
||||
$names = array_keys($data['specifics']);
|
||||
foreach ($names as $name) {
|
||||
// verify their content.
|
||||
$className = sprintf('FireflyIII\Import\Specifics\%s', $name);
|
||||
if (class_exists($className)) {
|
||||
$config['specifics'][$name] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $config;
|
||||
}
|
||||
}
|
||||
@ -23,8 +23,16 @@ declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Routine\File;
|
||||
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
|
||||
use FireflyIII\Models\Attachment;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Illuminate\Support\Collection;
|
||||
use League\Csv\Exception;
|
||||
use League\Csv\Reader;
|
||||
use League\Csv\Statement;
|
||||
use Log;
|
||||
|
||||
|
||||
/**
|
||||
@ -34,6 +42,8 @@ use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
*/
|
||||
class CSVProcessor implements FileProcessorInterface
|
||||
{
|
||||
/** @var AttachmentHelperInterface */
|
||||
private $attachments;
|
||||
/** @var ImportJob */
|
||||
private $importJob;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
@ -43,9 +53,33 @@ class CSVProcessor implements FileProcessorInterface
|
||||
* Fires the file processor.
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function run(): array
|
||||
{
|
||||
$config = $this->importJob->configuration;
|
||||
|
||||
// in order to actually map we also need to read the FULL file.
|
||||
try {
|
||||
$reader = $this->getReader();
|
||||
} catch (Exception $e) {
|
||||
Log::error($e->getMessage());
|
||||
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
|
||||
}
|
||||
// get mapping from config
|
||||
$roles = $config['column-roles'];
|
||||
|
||||
// get all lines from file:
|
||||
$lines = $this->getLines($reader, $config);
|
||||
|
||||
// make import objects, according to their role:
|
||||
$importables = $this->processLines($lines, $roles);
|
||||
|
||||
echo '<pre>';
|
||||
print_r($importables);
|
||||
print_r($lines);
|
||||
|
||||
exit;
|
||||
die('here we are');
|
||||
}
|
||||
|
||||
@ -54,9 +88,129 @@ class CSVProcessor implements FileProcessorInterface
|
||||
*/
|
||||
public function setJob(ImportJob $job): void
|
||||
{
|
||||
$this->importJob = $job;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->importJob = $job;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->attachments = app(AttachmentHelperInterface::class);
|
||||
$this->repository->setUser($job->user);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all lines from the CSV file.
|
||||
*
|
||||
* @param Reader $reader
|
||||
* @param array $config
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function getLines(Reader $reader, array $config): array
|
||||
{
|
||||
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
|
||||
try {
|
||||
$stmt = (new Statement)->offset($offset);
|
||||
} catch (Exception $e) {
|
||||
throw new FireflyException(sprintf('Could not execute statement: %s', $e->getMessage()));
|
||||
}
|
||||
$results = $stmt->process($reader);
|
||||
$lines = [];
|
||||
foreach ($results as $line) {
|
||||
$lines[] = array_values($line);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an instance of a CSV file reader so content of the file can be read.
|
||||
*
|
||||
* @throws \League\Csv\Exception
|
||||
*/
|
||||
private function getReader(): Reader
|
||||
{
|
||||
$content = '';
|
||||
/** @var Collection $collection */
|
||||
$collection = $this->importJob->attachments;
|
||||
/** @var Attachment $attachment */
|
||||
foreach ($collection as $attachment) {
|
||||
if ($attachment->filename === 'import_file') {
|
||||
$content = $this->attachments->getAttachmentContent($attachment);
|
||||
break;
|
||||
}
|
||||
}
|
||||
$config = $this->repository->getConfiguration($this->importJob);
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
|
||||
return $reader;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single column. Return is an array with:
|
||||
* [0 => key, 1 => value]
|
||||
* where the first item is the key under which the value
|
||||
* must be stored, and the second is the value.
|
||||
*
|
||||
* @param int $column
|
||||
* @param string $value
|
||||
* @param string $role
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function processColumn(int $column, string $value, string $role): array
|
||||
{
|
||||
switch ($role) {
|
||||
default:
|
||||
throw new FireflyException(sprintf('Cannot handle role "%s" with value "%s"', $role, $value));
|
||||
|
||||
|
||||
// feed each line into a new class which will process
|
||||
// the line.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process all lines in the CSV file. Each line is processed separately.
|
||||
*
|
||||
* @param array $lines
|
||||
* @param array $roles
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function processLines(array $lines, array $roles): array
|
||||
{
|
||||
$processed = [];
|
||||
foreach ($lines as $line) {
|
||||
$processed[] = $this->processSingleLine($line, $roles);
|
||||
|
||||
}
|
||||
|
||||
return $processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single line in the CSV file.
|
||||
* Each column is processed separately.
|
||||
*
|
||||
* @param array $line
|
||||
* @param array $roles
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function processSingleLine(array $line, array $roles): array
|
||||
{
|
||||
// todo run all specifics on row.
|
||||
$transaction = [];
|
||||
foreach ($line as $column => $value) {
|
||||
$value = trim($value);
|
||||
$role = $roles[$column] ?? '_ignore';
|
||||
[$key, $result] = $this->processColumn($column, $value, $role);
|
||||
// if relevant, find mapped value:
|
||||
}
|
||||
|
||||
return $transaction;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user