IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

First start for CSV file import.

Browse Source
This commit is contained in:
James Cole
2018-05-06 20:42:30 +02:00
parent a4524b3c2c
commit 1209f3b39a
13 changed files with 663 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/Http/Controllers/Import/JobStatusController.php
View File

@@ -116,13 +116,17 @@ class JobStatusController extends Controller
public function start(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['ready_to_run', 'need_job_config'];
$allowed = ['ready_to_run', 'need_job_config','error','running'];
// todo remove error and running.
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
Log::error('Job is not ready.');
// kill the job:
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "ready_to_run".']);
}
$importProvider = $importJob->provider;
$key = sprintf('import.routine.%s', $importProvider);
$className = config($key);

2
app/Import/Mapper/Bills.php
View File

@@ -43,7 +43,7 @@ class Bills implements MapperInterface
/** @var Bill $bill */
foreach ($result as $bill) {
$billId = (int)$bill->id;
$list[$billId] = $bill->name . ' [' . $bill->match . ']';
$list[$billId] = $bill->name;
}
asort($list);
$list = [0 => trans('import.map_do_not_map')] + $list;

42
app/Import/Routine/FakeRoutine.php
View File

@@ -37,19 +37,10 @@ use Log;
class FakeRoutine implements RoutineInterface
{
/** @var ImportJob */
private $job;
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* FakeRoutine constructor.
*/
public function __construct()
{
$this->repository = app(ImportJobRepositoryInterface::class);
}
/**
* Fake import routine has three stages:
*
@@ -63,49 +54,50 @@ class FakeRoutine implements RoutineInterface
*/
public function run(): void
{
Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->job->status));
if ($this->job->status !== 'running') {
Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->importJob->status));
if ($this->importJob->status !== 'running') {
throw new FireflyException('This fake job should not be started.'); // @codeCoverageIgnore
}
switch ($this->job->stage) {
switch ($this->importJob->stage) {
default:
throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->job->stage)); // @codeCoverageIgnore
throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->importJob->stage)); // @codeCoverageIgnore
case 'new':
/** @var StageNewHandler $handler */
$handler = app(StageNewHandler::class);
$handler->run();
$this->repository->setStage($this->job, 'ahoy');
$this->repository->setStage($this->importJob, 'ahoy');
// set job finished this step:
$this->repository->setStatus($this->job, 'ready_to_run');
$this->repository->setStatus($this->importJob, 'ready_to_run');
return;
case 'ahoy':
/** @var StageAhoyHandler $handler */
$handler = app(StageAhoyHandler::class);
$handler->run();
$this->repository->setStatus($this->job, 'need_job_config');
$this->repository->setStage($this->job, 'final');
$this->repository->setStatus($this->importJob, 'need_job_config');
$this->repository->setStage($this->importJob, 'final');
break;
case 'final':
/** @var StageFinalHandler $handler */
$handler = app(StageFinalHandler::class);
$handler->setJob($this->job);
$handler->setJob($this->importJob);
$transactions = $handler->getTransactions();
$this->repository->setStatus($this->job, 'provider_finished');
$this->repository->setStage($this->job, 'final');
$this->repository->setTransactions($this->job, $transactions);
$this->repository->setStatus($this->importJob, 'provider_finished');
$this->repository->setStage($this->importJob, 'final');
$this->repository->setTransactions($this->importJob, $transactions);
}
}
/**
* @param ImportJob $job
*
* @return mixed
* @return
*/
public function setJob(ImportJob $job)
public function setJob(ImportJob $job): void
{
$this->job = $job;
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
}

611
app/Import/Routine/FileRoutine.php
View File

@@ -22,292 +22,23 @@ declare(strict_types=1);
namespace FireflyIII\Import\Routine;
use Carbon\Carbon;
use DB;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use Illuminate\Support\Collection;
use FireflyIII\Support\Import\Routine\File\FileProcessorInterface;
use FireflyIII\Support\Import\Routine\File\FileRoutineInterface;
use Log;
/**
* @deprecated
* @codeCoverageIgnore
* Class FileRoutine
*/
class FileRoutine implements RoutineInterface
{
// /** @var Collection */
// public $errors;
// /** @var Collection */
// public $journals;
// /** @var int */
// public $lines = 0;
// /** @var ImportJob */
// private $job;
//
// /** @var ImportJobRepositoryInterface */
// private $repository;
//
// /**
// * ImportRoutine constructor.
// */
// public function __construct()
// {
// $this->journals = new Collection;
// $this->errors = new Collection;
// }
//
// /**
// * @return Collection
// */
// public function getErrors(): Collection
// {
// return $this->errors;
// }
//
// /**
// * @return Collection
// */
// public function getJournals(): Collection
// {
// return $this->journals;
// }
//
// /**
// * @return int
// */
// public function getLines(): int
// {
// return $this->lines;
// }
//
// /**
// *
// */
// public function run(): bool
// {
// if ('configured' !== $this->getStatus()) {
// Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
//
// return false;
// }
// set_time_limit(0);
// Log::info(sprintf('Start with import job %s', $this->job->key));
//
// // total steps: 6
// $this->setTotalSteps(6);
//
// $importObjects = $this->getImportObjects();
// $this->lines = $importObjects->count();
// $this->addStep();
//
// // total steps can now be extended. File has been scanned. 7 steps per line:
// $this->addTotalSteps(7 * $this->lines);
//
// // once done, use storage thing to actually store them:
// Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
//
// $storage = $this->storeObjects($importObjects);
// $this->addStep();
// Log::debug('Back in run()');
//
// Log::debug('Updated job...');
// Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
// $this->journals = $storage->journals;
// $this->errors = $storage->errors;
//
// Log::debug('Going to call createImportTag()');
//
// // create tag, link tag to all journals:
// $this->createImportTag();
// $this->addStep();
//
// // update job:
// $this->setStatus('finished');
//
// Log::info(sprintf('Done with import job %s', $this->job->key));
//
// return true;
// }
//
// /**
// * @param ImportJob $job
// */
// public function setJob(ImportJob $job)
// {
// $this->job = $job;
// $this->repository = app(ImportJobRepositoryInterface::class);
// $this->repository->setUser($job->user);
// }
//
// /**
// * @return Collection
// */
// protected function getImportObjects(): Collection
// {
// $objects = new Collection;
// $fileType = $this->getConfig()['file-type'] ?? 'csv';
// // will only respond to "file"
// $class = config(sprintf('import.options.file.processors.%s', $fileType));
// /** @var FileProcessorInterface $processor */
// $processor = app($class);
// $processor->setJob($this->job);
//
// if ('configured' === $this->getStatus()) {
// // set job as "running"...
// $this->setStatus('running');
//
// Log::debug('Job is configured, start with run()');
// $processor->run();
// $objects = $processor->getObjects();
// }
//
// return $objects;
// }
//
// /**
// * Shorthand method.
// */
// private function addStep()
// {
// $this->repository->addStepsDone($this->job, 1);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function addTotalSteps(int $steps)
// {
// $this->repository->addTotalSteps($this->job, $steps);
// }
//
// /**
// *
// */
// private function createImportTag(): Tag
// {
// Log::debug('Now in createImportTag()');
//
// if ($this->journals->count() < 1) {
// Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
//
// return new Tag;
// }
// $this->addTotalSteps($this->journals->count() + 2);
//
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $repository->setUser($this->job->user);
// $data = [
// 'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
// 'date' => new Carbon,
// 'description' => null,
// 'latitude' => null,
// 'longitude' => null,
// 'zoomLevel' => null,
// 'tagMode' => 'nothing',
// ];
// $tag = $repository->store($data);
// $this->addStep();
// $extended = $this->getExtendedStatus();
// $extended['tag'] = $tag->id;
// $this->setExtendedStatus($extended);
//
// Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
// Log::debug('Looping journals...');
// $journalIds = $this->journals->pluck('id')->toArray();
// $tagId = $tag->id;
// foreach ($journalIds as $journalId) {
// Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
// DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
// $this->addStep();
// }
// Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
// $this->addStep();
//
// return $tag;
// }
//
// /**
// * Shorthand method
// *
// * @return array
// */
// private function getConfig(): array
// {
// return $this->repository->getConfiguration($this->job);
// }
//
// /**
// * @return array
// */
// private function getExtendedStatus(): array
// {
// return $this->repository->getExtendedStatus($this->job);
// }
//
// /**
// * Shorthand method.
// *
// * @return string
// */
// private function getStatus(): string
// {
// return $this->repository->getStatus($this->job);
// }
//
// /**
// * @param array $extended
// */
// private function setExtendedStatus(array $extended): void
// {
// $this->repository->setExtendedStatus($this->job, $extended);
// }
//
// /**
// * Shorthand
// *
// * @param string $status
// */
// private function setStatus(string $status): void
// {
// $this->repository->setStatus($this->job, $status);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function setTotalSteps(int $steps)
// {
// $this->repository->setTotalSteps($this->job, $steps);
// }
//
// /**
// * @param Collection $objects
// *
// * @return ImportStorage
// */
// private function storeObjects(Collection $objects): ImportStorage
// {
// $config = $this->getConfig();
// $storage = new ImportStorage;
// $storage->setJob($this->job);
// $storage->setDateFormat($config['date-format']);
// $storage->setObjects($objects);
// $storage->store();
// Log::info('Back in storeObjects()');
//
// return $storage;
// }
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* At the end of each run(), the import routine must set the job to the expected status.
*
@@ -318,18 +49,334 @@ class FileRoutine implements RoutineInterface
*/
public function run(): void
{
// TODO: Implement run() method.
throw new NotImplementedException;
Log::debug(sprintf('Now in run() for file routine with status: %s', $this->importJob->status));
if ($this->importJob->status !== 'running') {
throw new FireflyException('This file import job should not be started.'); // @codeCoverageIgnore
}
switch ($this->importJob->stage) {
case 'ready_to_run':
// get processor, depending on file type
// is just CSV for now.
$processor = $this->getProcessor();
$transactions = $processor->run();
// make processor run.
// then done!
// move to status 'processor_finished'.
// $this->repository->setStatus($this->importJob, 'provider_finished');
// $this->repository->setStage($this->importJob, 'final');
break;
default:
throw new FireflyException(sprintf('Import routine cannot handle stage "%s"', $this->importJob->stage));
}
exit;
}
/**
* @param ImportJob $job
*
* @return mixed
* @return void
*/
public function setJob(ImportJob $job)
public function setJob(ImportJob $job): void
{
// TODO: Implement setJob() method.
throw new NotImplementedException;
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* Return the appropriate file routine handler for
* the file type of the job.
*
* @return FileProcessorInterface
*/
private function getProcessor(): FileProcessorInterface
{
$config = $this->repository->getConfiguration($this->importJob);
$type = $config['file-type'] ?? 'csv';
$class = config(sprintf('import.options.file.processors.%s', $type));
/** @var FileProcessorInterface $object */
$object = app($class);
return $object;
}
// /** @var Collection */
// public $errors;
// /** @var Collection */
// public $journals;
// /** @var int */
// public $lines = 0;
// /** @var ImportJob */
// private $job;
//
// /** @var ImportJobRepositoryInterface */
// private $repository;
//
// /**
// * ImportRoutine constructor.
// */
// public function __construct()
// {
// $this->journals = new Collection;
// $this->errors = new Collection;
// }
//
// /**
// * @return Collection
// */
// public function getErrors(): Collection
// {
// return $this->errors;
// }
//
// /**
// * @return Collection
// */
// public function getJournals(): Collection
// {
// return $this->journals;
// }
//
// /**
// * @return int
// */
// public function getLines(): int
// {
// return $this->lines;
// }
//
// /**
// *
// */
// public function run(): bool
// {
// if ('configured' !== $this->getStatus()) {
// Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
//
// return false;
// }
// set_time_limit(0);
// Log::info(sprintf('Start with import job %s', $this->job->key));
//
// // total steps: 6
// $this->setTotalSteps(6);
//
// $importObjects = $this->getImportObjects();
// $this->lines = $importObjects->count();
// $this->addStep();
//
// // total steps can now be extended. File has been scanned. 7 steps per line:
// $this->addTotalSteps(7 * $this->lines);
//
// // once done, use storage thing to actually store them:
// Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
//
// $storage = $this->storeObjects($importObjects);
// $this->addStep();
// Log::debug('Back in run()');
//
// Log::debug('Updated job...');
// Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
// $this->journals = $storage->journals;
// $this->errors = $storage->errors;
//
// Log::debug('Going to call createImportTag()');
//
// // create tag, link tag to all journals:
// $this->createImportTag();
// $this->addStep();
//
// // update job:
// $this->setStatus('finished');
//
// Log::info(sprintf('Done with import job %s', $this->job->key));
//
// return true;
// }
//
// /**
// * @param ImportJob $job
// */
// public function setJob(ImportJob $job)
// {
// $this->job = $job;
// $this->repository = app(ImportJobRepositoryInterface::class);
// $this->repository->setUser($job->user);
// }
//
// /**
// * @return Collection
// */
// protected function getImportObjects(): Collection
// {
// $objects = new Collection;
// $fileType = $this->getConfig()['file-type'] ?? 'csv';
// // will only respond to "file"
// $class = config(sprintf('import.options.file.processors.%s', $fileType));
// /** @var FileProcessorInterface $processor */
// $processor = app($class);
// $processor->setJob($this->job);
//
// if ('configured' === $this->getStatus()) {
// // set job as "running"...
// $this->setStatus('running');
//
// Log::debug('Job is configured, start with run()');
// $processor->run();
// $objects = $processor->getObjects();
// }
//
// return $objects;
// }
//
// /**
// * Shorthand method.
// */
// private function addStep()
// {
// $this->repository->addStepsDone($this->job, 1);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function addTotalSteps(int $steps)
// {
// $this->repository->addTotalSteps($this->job, $steps);
// }
//
// /**
// *
// */
// private function createImportTag(): Tag
// {
// Log::debug('Now in createImportTag()');
//
// if ($this->journals->count() < 1) {
// Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
//
// return new Tag;
// }
// $this->addTotalSteps($this->journals->count() + 2);
//
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $repository->setUser($this->job->user);
// $data = [
// 'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
// 'date' => new Carbon,
// 'description' => null,
// 'latitude' => null,
// 'longitude' => null,
// 'zoomLevel' => null,
// 'tagMode' => 'nothing',
// ];
// $tag = $repository->store($data);
// $this->addStep();
// $extended = $this->getExtendedStatus();
// $extended['tag'] = $tag->id;
// $this->setExtendedStatus($extended);
//
// Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
// Log::debug('Looping journals...');
// $journalIds = $this->journals->pluck('id')->toArray();
// $tagId = $tag->id;
// foreach ($journalIds as $journalId) {
// Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
// DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
// $this->addStep();
// }
// Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
// $this->addStep();
//
// return $tag;
// }
//
// /**
// * Shorthand method
// *
// * @return array
// */
// private function getConfig(): array
// {
// return $this->repository->getConfiguration($this->job);
// }
//
// /**
// * @return array
// */
// private function getExtendedStatus(): array
// {
// return $this->repository->getExtendedStatus($this->job);
// }
//
// /**
// * Shorthand method.
// *
// * @return string
// */
// private function getStatus(): string
// {
// return $this->repository->getStatus($this->job);
// }
//
// /**
// * @param array $extended
// */
// private function setExtendedStatus(array $extended): void
// {
// $this->repository->setExtendedStatus($this->job, $extended);
// }
//
// /**
// * Shorthand
// *
// * @param string $status
// */
// private function setStatus(string $status): void
// {
// $this->repository->setStatus($this->job, $status);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function setTotalSteps(int $steps)
// {
// $this->repository->setTotalSteps($this->job, $steps);
// }
//
// /**
// * @param Collection $objects
// *
// * @return ImportStorage
// */
// private function storeObjects(Collection $objects): ImportStorage
// {
// $config = $this->getConfig();
// $storage = new ImportStorage;
// $storage->setJob($this->job);
// $storage->setDateFormat($config['date-format']);
// $storage->setObjects($objects);
// $storage->store();
// Log::info('Back in storeObjects()');
//
// return $storage;
// }
}

4
app/Import/Routine/RoutineInterface.php
View File

@@ -43,7 +43,7 @@ interface RoutineInterface
/**
* @param ImportJob $job
*
* @return mixed
* @return void
*/
public function setJob(ImportJob $job);
public function setJob(ImportJob $job): void;
}

179
app/Support/Import/Configuration/File/ConfigureMappingHandler.php
View File

@@ -26,6 +26,8 @@ namespace FireflyIII\Support\Import\Configuration\File;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Import\Mapper\MapperInterface;
use FireflyIII\Import\MapperPreProcess\PreProcessorInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
@@ -33,6 +35,7 @@ use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag;
use League\Csv\Exception;
use League\Csv\Reader;
use League\Csv\Statement;
use Log;
/**
@@ -58,6 +61,22 @@ class ConfigureMappingHandler implements ConfigurationInterface
*/
public function configureJob(array $data): MessageBag
{
$config = $this->importJob->configuration;
if (isset($data['mapping']) && \is_array($data['mapping'])) {
foreach ($data['mapping'] as $index => $array) {
$config['column-mapping-config'][$index] = [];
foreach ($array as $value => $mapId) {
$mapId = (int)$mapId;
if (0 !== $mapId) {
$config['column-mapping-config'][$index][$value] = $mapId;
}
}
}
}
$this->repository->setConfiguration($this->importJob, $config);
$this->repository->setStage($this->importJob, 'ready_to_run');
return new MessageBag;
}
@@ -79,65 +98,11 @@ class ConfigureMappingHandler implements ConfigurationInterface
Log::error($e->getMessage());
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
}
//
// if ($config['has-headers']) {
// $offset = 1;
// }
// $stmt = (new Statement)->offset($offset);
// $results = $stmt->process($reader);
// $this->validSpecifics = array_keys(config('csv.import_specifics'));
// $indexes = array_keys($this->data);
// $rowIndex = 0;
// foreach ($results as $rowIndex => $row) {
// $row = $this->runSpecifics($row);
//
// //do something here
// foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
// if (!isset($row[$index])) {
// // don't really know how to handle this. Just skip, for now.
// continue;
// }
// $value = trim($row[$index]);
// if (\strlen($value) > 0) {
// // we can do some preprocessing here,
// // which is exclusively to fix the tags:
// if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) {
// /** @var PreProcessorInterface $preProcessor */
// $preProcessor = app($this->data[$index]['preProcessMap']);
// $result = $preProcessor->run($value);
// $this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result);
//
// Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
// Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
// Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]);
//
// continue;
// }
//
// $this->data[$index]['values'][] = $value;
// }
// }
// }
// $setIndexes = array_keys($this->data);
// foreach ($setIndexes as $index) {
// $this->data[$index]['values'] = array_unique($this->data[$index]['values']);
// asort($this->data[$index]['values']);
// // if the count of this array is zero, there is nothing to map.
// if (\count($this->data[$index]['values']) === 0) {
// unset($this->data[$index]);
// }
// }
// unset($setIndexes);
//
// // save number of rows, thus number of steps, in job:
// $steps = $rowIndex * 5;
// $extended = $this->job->extended_status;
// $extended['steps'] = $steps;
// $this->job->extended_status = $extended;
// $this->job->save();
//
// return $this->data;
// */
// get ALL values for the mappable columns from the CSV file:
$columnConfig = $this->getValuesForMapping($reader, $config, $columnConfig);
return $columnConfig;
}
/**
@@ -152,6 +117,36 @@ class ConfigureMappingHandler implements ConfigurationInterface
$this->columnConfig = [];
}
/**
* Apply the users selected specifics on the current row.
*
* @param array $config
* @param array $validSpecifics
* @param array $row
*
* @return array
*/
private function applySpecifics(array $config, array $validSpecifics, array $row): array
{
// run specifics here:
// and this is the point where the specifix go to work.
$specifics = $config['specifics'] ?? [];
$names = array_keys($specifics);
foreach ($names as $name) {
if (!\in_array($name, $validSpecifics)) {
continue;
}
$class = config(sprintf('csv.import_specifics.%s', $name));
/** @var SpecificInterface $specific */
$specific = app($class);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
return $row;
}
/**
* Create the "mapper" class that will eventually return the correct data for the user
* to map against. For example: a list of asset accounts. A list of budgets. A list of tags.
@@ -275,6 +270,72 @@ class ConfigureMappingHandler implements ConfigurationInterface
return $reader;
}
/**
* Read the CSV file. For each row, check for each column:
*
* - If it can be mapped. And if so,
* - Run the pre-processor
* - Add the value to the list of "values" that the user must map.
*
* @param Reader $reader
* @param array $columnConfig
*
* @return array
* @throws FireflyException
*/
private function getValuesForMapping(Reader $reader, array $config, array $columnConfig): array
{
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
} catch (Exception $e) {
throw new FireflyException(sprintf('Could not create reader: %s', $e->getMessage()));
}
$results = $stmt->process($reader);
$validSpecifics = array_keys(config('csv.import_specifics'));
$validIndexes = array_keys($columnConfig); // the actually columns that can be mapped.
foreach ($results as $rowIndex => $row) {
$row = $this->applySpecifics($config, $validSpecifics, $row);
//do something here
/** @var int $currentIndex */
foreach ($validIndexes as $currentIndex) { // this is simply 1, 2, 3, etc.
if (!isset($row[$currentIndex])) {
// don't need to handle this. Continue.
continue;
}
$value = trim($row[$currentIndex]);
if (\strlen($value) === 0) {
continue;
}
// we can do some preprocessing here,
// which is exclusively to fix the tags:
if (null !== $columnConfig[$currentIndex]['preProcessMap'] && \strlen($columnConfig[$currentIndex]['preProcessMap']) > 0) {
/** @var PreProcessorInterface $preProcessor */
$preProcessor = app($columnConfig[$currentIndex]['preProcessMap']);
$result = $preProcessor->run($value);
// can merge array, this is usually the case:
$columnConfig[$currentIndex]['values'] = array_merge($columnConfig[$currentIndex]['values'], $result);
continue;
}
$columnConfig[$currentIndex]['values'][] = $value;
}
}
// loop array again. This time, do uniqueness.
// and remove arrays that have 0 values.
foreach ($validIndexes as $currentIndex) {
$columnConfig[$currentIndex]['values'] = array_unique($columnConfig[$currentIndex]['values']);
asort($columnConfig[$currentIndex]['values']);
// if the count of this array is zero, there is nothing to map.
if (\count($columnConfig[$currentIndex]['values']) === 0) {
unset($columnConfig[$currentIndex]);
}
}
return $columnConfig;
}
/**
* For each given column name, will return either the name (when it's a valid one)
* or return the _ignore column.

70
app/Support/Import/Configuration/File/NewFileJobHandler.php
View File

@@ -25,7 +25,7 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\File;
use Crypt;
use FireflyIII\Console\Commands\Import;
use Exception;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
@@ -35,7 +35,6 @@ use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\MessageBag;
use Log;
use Storage;
use Exception;
/**
* Class NewFileJobHandler
@@ -50,37 +49,6 @@ class NewFileJobHandler implements ConfigurationInterface
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* Store data associated with current stage.
*
@@ -118,13 +86,47 @@ class NewFileJobHandler implements ConfigurationInterface
$this->storeConfig($attachment);
}
}
// set file type in config:
$config = $this->repository->getConfiguration($this->importJob);
$config['file-type'] = $data['import_file_type'];
$this->repository->setConfiguration($this->importJob, $config);
$this->repository->setStage($this->importJob, 'configure-upload');
return new MessageBag();
}
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* @param Attachment $attachment
*

62
app/Support/Import/Routine/File/CSVProcessor.php Normal file
View File

@@ -0,0 +1,62 @@
<?php
/**
* CSVProcessor.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
/**
* Class CSVProcessor
*
* @package FireflyIII\Support\Import\Routine\File
*/
class CSVProcessor implements FileProcessorInterface
{
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Fires the file processor.
*
* @return array
*/
public function run(): array
{
die('here we are');
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
}

49
app/Support/Import/Routine/File/FileProcessorInterface.php Normal file
View File

@@ -0,0 +1,49 @@
<?php
/**
* FileRoutineInterface.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Models\ImportJob;
/**
* Interface FileProcessorInterface
*
* @package FireflyIII\Support\Import\Routine\File
*/
interface FileProcessorInterface
{
/**
* Fires the file processor.
*
* @return array
*/
public function run(): array;
/**
* Set values.
*
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void;
}