From f04011f6a7a88ca4bccc8e9c3b6671eb9cfcbb8a Mon Sep 17 00:00:00 2001 From: James Cole Date: Wed, 7 Jun 2017 12:20:13 +0200 Subject: [PATCH 01/96] New translations firefly.php (Dutch) --- resources/lang/nl_NL/firefly.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lang/nl_NL/firefly.php b/resources/lang/nl_NL/firefly.php index 158bc9f235..14f8624a18 100644 --- a/resources/lang/nl_NL/firefly.php +++ b/resources/lang/nl_NL/firefly.php @@ -963,7 +963,7 @@ return [ 'split_this_transfer' => 'Splits deze overschrijving', 'cannot_edit_multiple_source' => 'Je kan transactie #:id met omschrijving ":description" niet splitsen, want deze bevat meerdere bronrekeningen.', 'cannot_edit_multiple_dest' => 'Je kan transactie #:id met omschrijving ":description" niet wijzigen, want deze bevat meerdere doelrekeningen.', - 'cannot_edit_opening_balance' => 'You cannot edit the opening balance of an account.', + 'cannot_edit_opening_balance' => 'Je kan het startsaldo van een rekening niet wijzigen via dit scherm.', 'no_edit_multiple_left' => 'Je hebt geen geldige transacties geselecteerd.', // import From b48de98865cf288a508b8292c26d680d1a2d0696 Mon Sep 17 00:00:00 2001 From: James Cole Date: Thu, 8 Jun 2017 10:35:02 +0200 Subject: [PATCH 02/96] Fix a bug where the balance routine forgot to account for accounts without a currency preference. --- app/Http/Controllers/Chart/AccountController.php | 1 + app/Support/Steam.php | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/app/Http/Controllers/Chart/AccountController.php b/app/Http/Controllers/Chart/AccountController.php index c0af003e81..3771f913a8 100644 --- a/app/Http/Controllers/Chart/AccountController.php +++ b/app/Http/Controllers/Chart/AccountController.php @@ -127,6 +127,7 @@ class AccountController extends Controller $chartData[$account->name] = $diff; } } + arsort($chartData); $data = $this->generator->singleSet(strval(trans('firefly.spent')), $chartData); $cache->store($data); diff --git a/app/Support/Steam.php b/app/Support/Steam.php index 398400e05b..7294b07bde 100644 --- a/app/Support/Steam.php +++ b/app/Support/Steam.php @@ -13,6 +13,7 @@ declare(strict_types=1); namespace FireflyIII\Support; +use Amount; use Carbon\Carbon; use Crypt; use DB; @@ -47,6 +48,11 @@ class Steam return $cache->get(); // @codeCoverageIgnore } $currencyId = intval($account->getMeta('currency_id')); + // if null, use system default currency: + if ($currencyId === 0) { + $currency = Amount::getDefaultCurrency(); + $currencyId = $currency->id; + } // first part: get all balances in own currency: $nativeBalance = strval( $account->transactions() From a2145f6b490e77422d02aed0600fdb6c1732d212 Mon Sep 17 00:00:00 2001 From: James Cole Date: Thu, 8 Jun 2017 10:54:15 +0200 Subject: [PATCH 03/96] Possible fix for #667 --- app/Console/Commands/UpgradeDatabase.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/Console/Commands/UpgradeDatabase.php b/app/Console/Commands/UpgradeDatabase.php index 107f61f604..42566a6cee 100644 --- a/app/Console/Commands/UpgradeDatabase.php +++ b/app/Console/Commands/UpgradeDatabase.php @@ -316,6 +316,7 @@ class UpgradeDatabase extends Command $notification = '%s #%d uses %s but should use %s. It has been updated. Please verify this in Firefly III.'; $transfer = 'Transfer #%d has been updated to use the correct currencies. Please verify this in Firefly III.'; $driver = DB::connection()->getDriverName(); + $pgsql = ['pgsql', 'postgresql']; foreach ($types as $type => $operator) { $query = TransactionJournal @@ -328,10 +329,10 @@ class UpgradeDatabase extends Command ->leftJoin('account_meta', 'account_meta.account_id', '=', 'accounts.id') ->where('transaction_types.type', $type) ->where('account_meta.name', 'currency_id'); - if ($driver === 'postgresql') { + if (in_array($driver, $pgsql)) { $query->where('transaction_journals.transaction_currency_id', '!=', DB::raw('cast(account_meta.data as int)')); } - if ($driver !== 'postgresql') { + if (!in_array($driver, $pgsql)) { $query->where('transaction_journals.transaction_currency_id', '!=', DB::raw('account_meta.data')); } From 762d7bcc34a85ac4e65adf4fe5811fcc717dd268 Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 9 Jun 2017 11:51:59 +0200 Subject: [PATCH 04/96] Fix database for postgresql --- database/migrations/2016_06_16_000002_create_main_tables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/migrations/2016_06_16_000002_create_main_tables.php b/database/migrations/2016_06_16_000002_create_main_tables.php index 0f5fe8d4e4..300024918a 100644 --- a/database/migrations/2016_06_16_000002_create_main_tables.php +++ b/database/migrations/2016_06_16_000002_create_main_tables.php @@ -474,7 +474,7 @@ class CreateMainTables extends Migration $table->text('description')->nullable(); $table->decimal('latitude', 24, 12)->nullable(); $table->decimal('longitude', 24, 12)->nullable(); - $table->boolean('zoomLevel')->nullable(); + $table->smallInteger('zoomLevel', false, true)->nullable(); // link user id to users table $table->foreign('user_id')->references('id')->on('users')->onDelete('cascade'); From 1f9b7faa6097c9a523b090099a57d192d1ed2af7 Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 9 Jun 2017 11:52:20 +0200 Subject: [PATCH 05/96] Code for #660 --- app/Import/ImportStorage.php | 20 +++++++++++--------- app/Support/Import/CsvImportSupportTrait.php | 3 +++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/Import/ImportStorage.php b/app/Import/ImportStorage.php index 2d5524fcec..37808b6394 100644 --- a/app/Import/ImportStorage.php +++ b/app/Import/ImportStorage.php @@ -361,17 +361,19 @@ class ImportStorage // create new transactions. This is something that needs a rewrite for multiple/split transactions. $sourceData = [ - 'account_id' => $accounts['source']->id, - 'transaction_journal_id' => $journal->id, - 'description' => null, - 'amount' => bcmul($amount, '-1'), + 'account_id' => $accounts['source']->id, + 'transaction_journal_id' => $journal->id, + 'transaction_currency_id' => $journal->transaction_currency_id, + 'description' => null, + 'amount' => bcmul($amount, '-1'), ]; $destinationData = [ - 'account_id' => $accounts['destination']->id, - 'transaction_journal_id' => $journal->id, - 'description' => null, - 'amount' => $amount, + 'account_id' => $accounts['destination']->id, + 'transaction_currency_id' => $journal->transaction_currency_id, + 'transaction_journal_id' => $journal->id, + 'description' => null, + 'amount' => $amount, ]; $one = Transaction::create($sourceData); @@ -383,7 +385,7 @@ class ImportStorage } if (is_null($two->id)) { - Log::error('Could not create transaction 1.', $two->getErrors()->all()); + Log::error('Could not create transaction 2.', $two->getErrors()->all()); $error = true; } diff --git a/app/Support/Import/CsvImportSupportTrait.php b/app/Support/Import/CsvImportSupportTrait.php index efee16704a..dbf915f50f 100644 --- a/app/Support/Import/CsvImportSupportTrait.php +++ b/app/Support/Import/CsvImportSupportTrait.php @@ -16,12 +16,15 @@ use FireflyIII\Exceptions\FireflyException; use FireflyIII\Import\Mapper\MapperInterface; use FireflyIII\Import\MapperPreProcess\PreProcessorInterface; use FireflyIII\Import\Specifics\SpecificInterface; +use FireflyIII\Models\ImportJob; use League\Csv\Reader; use Log; /** * Trait CsvImportSupportTrait * + * @property ImportJob $job + * * @package FireflyIII\Support\Import */ trait CsvImportSupportTrait From 0b4efe4ae1042c4cdf68d6ac37572883d0b71871 Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 9 Jun 2017 12:53:31 +0200 Subject: [PATCH 06/96] Small typo in chart. [skip ci] --- app/Http/Controllers/Chart/AccountController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/Chart/AccountController.php b/app/Http/Controllers/Chart/AccountController.php index 3771f913a8..61194faff9 100644 --- a/app/Http/Controllers/Chart/AccountController.php +++ b/app/Http/Controllers/Chart/AccountController.php @@ -425,7 +425,7 @@ class AccountController extends Controller } arsort($chartData); - $data = $this->generator->singleSet(strval(trans('firefly.spent')), $chartData); + $data = $this->generator->singleSet(strval(trans('firefly.earned')), $chartData); $cache->store($data); return Response::json($data); From 091596e80eb45156ea3592921c93bba5fcb1b477 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 10 Jun 2017 15:09:41 +0200 Subject: [PATCH 07/96] Lots of new code for new importer routine. --- app/Http/Controllers/ImportController.php | 561 ++++++++---------- app/Http/Requests/ImportUploadRequest.php | 5 +- .../Configurator/ConfiguratorInterface.php | 60 ++ app/Import/Configurator/CsvConfigurator.php | 140 +++++ .../{CsvImporter.php => OldCsvImporter.php} | 0 ...Interface.php => OldImporterInterface.php} | 0 app/Import/Setup/CsvSetup.php | 1 - app/Models/ImportJob.php | 8 +- .../ImportJob/ImportJobRepository.php | 79 ++- .../ImportJobRepositoryInterface.php | 17 + .../Configuration/ConfigurationInterface.php | 46 ++ .../Import/Configuration/Csv/Initial.php | 122 ++++ app/Support/Import/Configuration/Csv/Map.php | 229 +++++++ .../Import/Configuration/Csv/Roles.php | 261 ++++++++ config/csv.php | 1 + config/firefly.php | 5 +- public/js/ff/import/status.js | 103 +++- resources/lang/en_US/csv.php | 46 +- resources/lang/en_US/firefly.php | 1 - .../csv/{configure.twig => initial.twig} | 47 +- resources/views/import/csv/map.twig | 5 +- resources/views/import/csv/roles.twig | 36 +- resources/views/import/index.twig | 5 +- resources/views/import/status.twig | 55 +- routes/web.php | 5 +- 25 files changed, 1415 insertions(+), 423 deletions(-) create mode 100644 app/Import/Configurator/ConfiguratorInterface.php create mode 100644 app/Import/Configurator/CsvConfigurator.php rename app/Import/Importer/{CsvImporter.php => OldCsvImporter.php} (100%) rename app/Import/Importer/{ImporterInterface.php => OldImporterInterface.php} (100%) create mode 100644 app/Support/Import/Configuration/ConfigurationInterface.php create mode 100644 app/Support/Import/Configuration/Csv/Initial.php create mode 100644 app/Support/Import/Configuration/Csv/Map.php create mode 100644 app/Support/Import/Configuration/Csv/Roles.php rename resources/views/import/csv/{configure.twig => initial.twig} (61%) diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index 325698f720..a9fb1491b4 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -12,11 +12,10 @@ declare(strict_types=1); namespace FireflyIII\Http\Controllers; -use Crypt; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Http\Requests\ImportUploadRequest; +use FireflyIII\Import\Configurator\ConfiguratorInterface; use FireflyIII\Import\ImportProcedureInterface; -use FireflyIII\Import\Setup\SetupInterface; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Repositories\Tag\TagRepositoryInterface; @@ -25,10 +24,6 @@ use Illuminate\Http\Request; use Illuminate\Http\Response as LaravelResponse; use Log; use Response; -use Session; -use SplFileObject; -use Storage; -use Symfony\Component\HttpFoundation\File\UploadedFile; use View; /** @@ -54,30 +49,28 @@ class ImportController extends Controller } ); } + // + // /** + // * This is the last step before the import starts. + // * + // * @param ImportJob $job + // * + // * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View + // */ + // public function complete(ImportJob $job) + // { + // Log::debug('Now in complete()', ['job' => $job->key]); + // if (!$this->jobInCorrectStep($job, 'complete')) { + // return $this->redirectToCorrectStep($job); + // } + // $subTitle = trans('firefly.import_complete'); + // $subTitleIcon = 'fa-star'; + // + // return view('import.complete', compact('job', 'subTitle', 'subTitleIcon')); + // } /** - * This is the last step before the import starts. - * - * @param ImportJob $job - * - * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View - */ - public function complete(ImportJob $job) - { - Log::debug('Now in complete()', ['job' => $job->key]); - if (!$this->jobInCorrectStep($job, 'complete')) { - return $this->redirectToCorrectStep($job); - } - $subTitle = trans('firefly.import_complete'); - $subTitleIcon = 'fa-star'; - - return view('import.complete', compact('job', 'subTitle', 'subTitleIcon')); - } - - /** - * This is step 3. - * This is the first step in configuring the job. It can only be executed - * when the job is set to "import_status_never_started". + * This is step 3. This repeats until the job is configured. * * @param ImportJob $job * @@ -86,19 +79,19 @@ class ImportController extends Controller */ public function configure(ImportJob $job) { - Log::debug('Now at start of configure()'); - if (!$this->jobInCorrectStep($job, 'configure')) { - return $this->redirectToCorrectStep($job); - } + // create configuration class: + $configurator = $this->makeConfigurator($job); - // actual code - $importer = $this->makeImporter($job); - $importer->configure(); - $data = $importer->getConfigurationData(); + // is the job already configured? + if ($configurator->isJobConfigured()) { + return redirect(route('import.status', [$job->key])); + } + $view = $configurator->getNextView(); + $data = $configurator->getNextData(); $subTitle = trans('firefly.configure_import'); $subTitleIcon = 'fa-wrench'; - return view('import.' . $job->file_type . '.configure', compact('data', 'job', 'subTitle', 'subTitleIcon')); + return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon')); } /** @@ -134,25 +127,25 @@ class ImportController extends Controller } - /** - * @param ImportJob $job - * - * @return View - */ - public function finished(ImportJob $job) - { - if (!$this->jobInCorrectStep($job, 'finished')) { - return $this->redirectToCorrectStep($job); - } - - // if there is a tag (there might not be), we can link to it: - $tagId = $job->extended_status['importTag'] ?? 0; - - $subTitle = trans('firefly.import_finished'); - $subTitleIcon = 'fa-star'; - - return view('import.finished', compact('job', 'subTitle', 'subTitleIcon', 'tagId')); - } + // /** + // * @param ImportJob $job + // * + // * @return View + // */ + // public function finished(ImportJob $job) + // { + // if (!$this->jobInCorrectStep($job, 'finished')) { + // return $this->redirectToCorrectStep($job); + // } + // + // // if there is a tag (there might not be), we can link to it: + // $tagId = $job->extended_status['importTag'] ?? 0; + // + // $subTitle = trans('firefly.import_finished'); + // $subTitleIcon = 'fa-star'; + // + // return view('import.finished', compact('job', 'subTitle', 'subTitleIcon', 'tagId')); + // } /** * This is step 1. Upload a file. @@ -161,7 +154,6 @@ class ImportController extends Controller */ public function index() { - Log::debug('Now at index'); $subTitle = trans('firefly.import_data_index'); $subTitleIcon = 'fa-home'; $importFileTypes = []; @@ -174,6 +166,35 @@ class ImportController extends Controller return view('import.index', compact('subTitle', 'subTitleIcon', 'importFileTypes', 'defaultImportType')); } + /** + * This is step 2. It creates an Import Job. Stores the import. + * + * @param ImportUploadRequest $request + * @param ImportJobRepositoryInterface $repository + * @param UserRepositoryInterface $userRepository + * + * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector + */ + public function initialize(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository) + { + Log::debug('Now in initialize()'); + + // create import job: + $type = $request->get('import_file_type'); + $job = $repository->create($type); + Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]); + + // process file: + $repository->processFile($job, $request->files->get('import_file')); + + // process config, if present: + if ($request->files->has('configuration_file')) { + $repository->processConfiguration($job, $request->files->get('configuration_file')); + } + + return redirect(route('import.configure', [$job->key])); + } + /** * @param ImportJob $job * @@ -182,22 +203,22 @@ class ImportController extends Controller public function json(ImportJob $job) { $result = [ - 'showPercentage' => false, - 'started' => false, - 'finished' => false, - 'running' => false, - 'errors' => $job->extended_status['errors'], - 'percentage' => 0, - 'steps' => $job->extended_status['total_steps'], - 'stepsDone' => $job->extended_status['steps_done'], - 'statusText' => trans('firefly.import_status_' . $job->status), - 'finishedText' => '', + 'started' => false, + 'finished' => false, + 'running' => false, + 'errors' => $job->extended_status['errors'], + 'percentage' => 0, + 'steps' => $job->extended_status['total_steps'], + 'stepsDone' => $job->extended_status['steps_done'], + 'statusText' => trans('firefly.import_status_' . $job->status), + 'status' => $job->status, + 'finishedText' => '', ]; $percentage = 0; if ($job->extended_status['total_steps'] !== 0) { $percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0); } - if ($job->status === 'import_complete') { + if ($job->status === 'complete') { $tagId = $job->extended_status['importTag']; /** @var TagRepositoryInterface $repository */ $repository = app(TagRepositoryInterface::class); @@ -206,11 +227,10 @@ class ImportController extends Controller $result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]); } - if ($job->status === 'import_running') { - $result['started'] = true; - $result['running'] = true; - $result['showPercentage'] = true; - $result['percentage'] = $percentage; + if ($job->status === 'running') { + $result['started'] = true; + $result['running'] = true; + $result['percentage'] = $percentage; } return Response::json($result); @@ -228,87 +248,81 @@ class ImportController extends Controller public function postConfigure(Request $request, ImportJobRepositoryInterface $repository, ImportJob $job) { Log::debug('Now in postConfigure()', ['job' => $job->key]); - if (!$this->jobInCorrectStep($job, 'process')) { - return $this->redirectToCorrectStep($job); + $configurator = $this->makeConfigurator($job); + + // is the job already configured? + if ($configurator->isJobConfigured()) { + return redirect(route('import.status', [$job->key])); } - Log::debug('Continue postConfigure()', ['job' => $job->key]); + $data = $request->all(); + $configurator->configureJob($data); - // actual code - $importer = $this->makeImporter($job); - $data = $request->all(); - $files = $request->files; - $importer->saveImportConfiguration($data, $files); - - // update job: - $repository->updateStatus($job, 'import_configuration_saved'); - - // return redirect to settings. - // this could loop until the user is done. - return redirect(route('import.settings', [$job->key])); + // return to configure + return redirect(route('import.configure', [$job->key])); } - /** - * This step 6. Depending on the importer, this will process the - * settings given and store them. - * - * @param Request $request - * @param ImportJob $job - * - * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector - * @throws FireflyException - */ - public function postSettings(Request $request, ImportJob $job) - { - Log::debug('Now in postSettings()', ['job' => $job->key]); - if (!$this->jobInCorrectStep($job, 'store-settings')) { - return $this->redirectToCorrectStep($job); - } - $importer = $this->makeImporter($job); - $importer->storeSettings($request); + // /** + // * This step 6. Depending on the importer, this will process the + // * settings given and store them. + // * + // * @param Request $request + // * @param ImportJob $job + // * + // * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector + // * @throws FireflyException + // */ + // public function postSettings(Request $request, ImportJob $job) + // { + // Log::debug('Now in postSettings()', ['job' => $job->key]); + // if (!$this->jobInCorrectStep($job, 'store-settings')) { + // return $this->redirectToCorrectStep($job); + // } + // $importer = $this->makeImporter($job); + // $importer->storeSettings($request); + // + // // return redirect to settings (for more settings perhaps) + // return redirect(route('import.settings', [$job->key])); + // } - // return redirect to settings (for more settings perhaps) - return redirect(route('import.settings', [$job->key])); - } - - /** - * Step 5. Depending on the importer, this will show the user settings to - * fill in. - * - * @param ImportJobRepositoryInterface $repository - * @param ImportJob $job - * - * @return View - */ - public function settings(ImportJobRepositoryInterface $repository, ImportJob $job) - { - Log::debug('Now in settings()', ['job' => $job->key]); - if (!$this->jobInCorrectStep($job, 'settings')) { - return $this->redirectToCorrectStep($job); - } - Log::debug('Continue in settings()'); - $importer = $this->makeImporter($job); - $subTitle = trans('firefly.settings_for_import'); - $subTitleIcon = 'fa-wrench'; - - // now show settings screen to user. - if ($importer->requireUserSettings()) { - Log::debug('Job requires user config.'); - $data = $importer->getDataForSettings(); - $view = $importer->getViewForSettings(); - - return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon')); - } - Log::debug('Job does NOT require user config.'); - - $repository->updateStatus($job, 'settings_complete'); - - // if no more settings, save job and continue to process thing. - return redirect(route('import.complete', [$job->key])); - - // ask the importer for the requested action. - // for example pick columns or map data. - // depends of course on the data in the job. - } + // /** + // * Step 5. Depending on the importer, this will show the user settings to + // * fill in. + // * + // * @param ImportJobRepositoryInterface $repository + // * @param ImportJob $job + // * + // * @return View + // */ + // public function settings(ImportJobRepositoryInterface $repository, ImportJob $job) + // { + // Log::debug('Now in settings()', ['job' => $job->key]); + // if (!$this->jobInCorrectStep($job, 'settings')) { + // return $this->redirectToCorrectStep($job); + // } + // Log::debug('Continue in settings()'); + // $importer = $this->makeImporter($job); + // $subTitle = trans('firefly.settings_for_import'); + // $subTitleIcon = 'fa-wrench'; + // + // // now show settings screen to user. + // if ($importer->requireUserSettings()) { + // Log::debug('Job requires user config.'); + // $data = $importer->getDataForSettings(); + // $view = $importer->getViewForSettings(); + // + // return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon')); + // } + // Log::debug('Job does NOT require user config.'); + // + // $repository->updateStatus($job, 'settings_complete'); + // + // // if no more settings, save job and continue to process thing. + // return redirect(route('import.complete', [$job->key])); + // + // // ask the importer for the requested action. + // // for example pick columns or map data. + // // depends of course on the data in the job. + // } /** * @param ImportProcedureInterface $importProcedure @@ -316,6 +330,7 @@ class ImportController extends Controller */ public function start(ImportProcedureInterface $importProcedure, ImportJob $job) { + die('TODO here.'); set_time_limit(0); if ($job->status == 'settings_complete') { $importProcedure->runImport($job); @@ -330,175 +345,117 @@ class ImportController extends Controller * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View */ public function status(ImportJob $job) - { // - Log::debug('Now in status()', ['job' => $job->key]); - if (!$this->jobInCorrectStep($job, 'status')) { - return $this->redirectToCorrectStep($job); - } + { $subTitle = trans('firefly.import_status'); $subTitleIcon = 'fa-star'; return view('import.status', compact('job', 'subTitle', 'subTitleIcon')); } - - /** - * This is step 2. It creates an Import Job. Stores the import. - * - * @param ImportUploadRequest $request - * @param ImportJobRepositoryInterface $repository - * @param UserRepositoryInterface $userRepository - * - * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector - */ - public function upload(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository) - { - Log::debug('Now in upload()'); - // create import job: - $type = $request->get('import_file_type'); - $job = $repository->create($type); - Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]); - - /** @var UploadedFile $upload */ - $upload = $request->files->get('import_file'); - $newName = $job->key . '.upload'; - $uploaded = new SplFileObject($upload->getRealPath()); - $content = $uploaded->fread($uploaded->getSize()); - $contentEncrypted = Crypt::encrypt($content); - $disk = Storage::disk('upload'); - - // user is demo user, replace upload with prepared file. - if ($userRepository->hasRole(auth()->user(), 'demo')) { - $stubsDisk = Storage::disk('stubs'); - $content = $stubsDisk->get('demo-import.csv'); - $contentEncrypted = Crypt::encrypt($content); - $disk->put($newName, $contentEncrypted); - Log::debug('Replaced upload with demo file.'); - - // also set up prepared configuration. - $configuration = json_decode($stubsDisk->get('demo-configuration.json'), true); - $repository->setConfiguration($job, $configuration); - Log::debug('Set configuration for demo user', $configuration); - - // also flash info - Session::flash('info', trans('demo.import-configure-security')); - } - if (!$userRepository->hasRole(auth()->user(), 'demo')) { - // user is not demo, process original upload: - $disk->put($newName, $contentEncrypted); - Log::debug('Uploaded file', ['name' => $upload->getClientOriginalName(), 'size' => $upload->getSize(), 'mime' => $upload->getClientMimeType()]); - } - - // store configuration file's content into the job's configuration thing. Otherwise, leave it empty. - // demo user's configuration upload is ignored completely. - if ($request->files->has('configuration_file') && !auth()->user()->hasRole('demo')) { - /** @var UploadedFile $configFile */ - $configFile = $request->files->get('configuration_file'); - Log::debug( - 'Uploaded configuration file', - ['name' => $configFile->getClientOriginalName(), 'size' => $configFile->getSize(), 'mime' => $configFile->getClientMimeType()] - ); - - $configFileObject = new SplFileObject($configFile->getRealPath()); - $configRaw = $configFileObject->fread($configFileObject->getSize()); - $configuration = json_decode($configRaw, true); - - // @codeCoverageIgnoreStart - if (!is_null($configuration) && is_array($configuration)) { - Log::debug('Found configuration', $configuration); - $repository->setConfiguration($job, $configuration); - } - // @codeCoverageIgnoreEnd - } - - return redirect(route('import.configure', [$job->key])); - } - - /** - * @param ImportJob $job - * @param string $method - * - * @return bool - */ - private function jobInCorrectStep(ImportJob $job, string $method): bool - { - Log::debug('Now in jobInCorrectStep()', ['job' => $job->key, 'method' => $method]); - switch ($method) { - case 'configure': - case 'process': - return $job->status === 'import_status_never_started'; - case 'settings': - case 'store-settings': - Log::debug(sprintf('Job %d with key %s has status %s', $job->id, $job->key, $job->status)); - - return $job->status === 'import_configuration_saved'; - case 'finished': - return $job->status === 'import_complete'; - case 'complete': - return $job->status === 'settings_complete'; - case 'status': - return ($job->status === 'settings_complete') || ($job->status === 'import_running'); - } - - return false; // @codeCoverageIgnore - - } + // /** + // * @param ImportJob $job + // * @param string $method + // * + // * @return bool + // */ + // private function jobInCorrectStep(ImportJob $job, string $method): bool + // { + // Log::debug('Now in jobInCorrectStep()', ['job' => $job->key, 'method' => $method]); + // switch ($method) { + // case 'configure': + // case 'process': + // return $job->status === 'import_status_never_started'; + // case 'settings': + // case 'store-settings': + // Log::debug(sprintf('Job %d with key %s has status %s', $job->id, $job->key, $job->status)); + // + // return $job->status === 'import_configuration_saved'; + // case 'finished': + // return $job->status === 'import_complete'; + // case 'complete': + // return $job->status === 'settings_complete'; + // case 'status': + // return ($job->status === 'settings_complete') || ($job->status === 'import_running'); + // } + // + // return false; // @codeCoverageIgnore + // + // } /** * @param ImportJob $job * - * @return SetupInterface + * @return ConfiguratorInterface * @throws FireflyException */ - private function makeImporter(ImportJob $job): SetupInterface + private function makeConfigurator(ImportJob $job): ConfiguratorInterface { - // create proper importer (depends on job) - $type = strtolower($job->file_type); - - // validate type: - $validTypes = array_keys(config('firefly.import_formats')); - - - if (in_array($type, $validTypes)) { - /** @var SetupInterface $importer */ - $importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup'); - $importer->setJob($job); - - return $importer; + $type = $job->file_type; + $key = sprintf('firefly.import_configurators.%s', $type); + $className = config($key); + if (is_null($className)) { + throw new FireflyException('Cannot find configurator class for this job.'); } - throw new FireflyException(sprintf('"%s" is not a valid file type', $type)); // @codeCoverageIgnore + $configurator = new $className($job); + return $configurator; } - /** - * @param ImportJob $job - * - * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector - * @throws FireflyException - */ - private function redirectToCorrectStep(ImportJob $job) - { - Log::debug('Now in redirectToCorrectStep()', ['job' => $job->key]); - switch ($job->status) { - case 'import_status_never_started': - Log::debug('Will redirect to configure()'); + // /** + // * @param ImportJob $job + // * + // * @return SetupInterface + // * @throws FireflyException + // */ + // private function makeImporter(ImportJob $job): SetupInterface + // { + // // create proper importer (depends on job) + // $type = strtolower($job->file_type); + // + // // validate type: + // $validTypes = array_keys(config('firefly.import_formats')); + // + // + // if (in_array($type, $validTypes)) { + // /** @var SetupInterface $importer */ + // $importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup'); + // $importer->setJob($job); + // + // return $importer; + // } + // throw new FireflyException(sprintf('"%s" is not a valid file type', $type)); // @codeCoverageIgnore + // + // } - return redirect(route('import.configure', [$job->key])); - case 'import_configuration_saved': - Log::debug('Will redirect to settings()'); - - return redirect(route('import.settings', [$job->key])); - case 'settings_complete': - Log::debug('Will redirect to complete()'); - - return redirect(route('import.complete', [$job->key])); - case 'import_complete': - Log::debug('Will redirect to finished()'); - - return redirect(route('import.finished', [$job->key])); - } - - throw new FireflyException('Cannot redirect for job state ' . $job->status); // @codeCoverageIgnore - - } + // /** + // * @param ImportJob $job + // * + // * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector + // * @throws FireflyException + // */ + // private function redirectToCorrectStep(ImportJob $job) + // { + // Log::debug('Now in redirectToCorrectStep()', ['job' => $job->key]); + // switch ($job->status) { + // case 'import_status_never_started': + // Log::debug('Will redirect to configure()'); + // + // return redirect(route('import.configure', [$job->key])); + // case 'import_configuration_saved': + // Log::debug('Will redirect to settings()'); + // + // return redirect(route('import.settings', [$job->key])); + // case 'settings_complete': + // Log::debug('Will redirect to complete()'); + // + // return redirect(route('import.complete', [$job->key])); + // case 'import_complete': + // Log::debug('Will redirect to finished()'); + // + // return redirect(route('import.finished', [$job->key])); + // } + // + // throw new FireflyException('Cannot redirect for job state ' . $job->status); // @codeCoverageIgnore + // + // } } diff --git a/app/Http/Requests/ImportUploadRequest.php b/app/Http/Requests/ImportUploadRequest.php index 93065bcada..758ee6164a 100644 --- a/app/Http/Requests/ImportUploadRequest.php +++ b/app/Http/Requests/ImportUploadRequest.php @@ -38,8 +38,9 @@ class ImportUploadRequest extends Request $types = array_keys(config('firefly.import_formats')); return [ - 'import_file' => 'required|file', - 'import_file_type' => 'required|in:' . join(',', $types), + 'import_file' => 'required|file', + 'import_file_type' => 'required|in:' . join(',', $types), + 'configuration_file' => 'file', ]; } diff --git a/app/Import/Configurator/ConfiguratorInterface.php b/app/Import/Configurator/ConfiguratorInterface.php new file mode 100644 index 0000000000..2de21661df --- /dev/null +++ b/app/Import/Configurator/ConfiguratorInterface.php @@ -0,0 +1,60 @@ +job = $job; + if (is_null($this->job->configuration) || count($this->job->configuration) === 0) { + Log::debug(sprintf('Gave import job %s initial configuration.', $this->job->key)); + $this->job->configuration = config('csv.default_config'); + $this->job->save(); + } + } + + /** + * Store any data from the $data array into the job. + * + * @param array $data + * + * @return bool + * @throws FireflyException + */ + public function configureJob(array $data): bool + { + $class = $this->getConfigurationClass(); + + /** @var ConfigurationInterface $object */ + $object = new $class($this->job); + + return $object->storeConfiguration($data); + } + + /** + * Return the data required for the next step in the job configuration. + * + * @return array + * @throws FireflyException + */ + public function getNextData(): array + { + $class = $this->getConfigurationClass(); + + /** @var ConfigurationInterface $object */ + $object = new $class($this->job); + + return $object->getData(); + + } + + /** + * @return string + * @throws FireflyException + */ + public function getNextView(): string + { + if (!$this->job->configuration['initial-config-complete']) { + return 'import.csv.initial'; + } + if (!$this->job->configuration['column-roles-complete']) { + return 'import.csv.roles'; + } + if (!$this->job->configuration['column-mapping-complete']) { + return 'import.csv.map'; + } + + throw new FireflyException('No view for state'); + } + + /** + * @return bool + */ + public function isJobConfigured(): bool + { + if ($this->job->configuration['initial-config-complete'] + && $this->job->configuration['column-roles-complete'] + && $this->job->configuration['column-mapping-complete'] + ) { + $this->job->status = 'configured'; + $this->job->save(); + + return true; + } + + return false; + } + + /** + * @return string + * @throws FireflyException + */ + private function getConfigurationClass(): string + { + $class = false; + switch (true) { + case(!$this->job->configuration['initial-config-complete']): + $class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Initial'; + break; + case (!$this->job->configuration['column-roles-complete']): + $class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Roles'; + break; + case (!$this->job->configuration['column-mapping-complete']): + $class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Map'; + break; + default: + break; + } + + if ($class === false) { + throw new FireflyException('Cannot handle current job state in getConfigurationClass().'); + } + if (!class_exists($class)) { + throw new FireflyException(sprintf('Class %s does not exist in getConfigurationClass().', $class)); + } + + return $class; + } +} \ No newline at end of file diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/OldCsvImporter.php similarity index 100% rename from app/Import/Importer/CsvImporter.php rename to app/Import/Importer/OldCsvImporter.php diff --git a/app/Import/Importer/ImporterInterface.php b/app/Import/Importer/OldImporterInterface.php similarity index 100% rename from app/Import/Importer/ImporterInterface.php rename to app/Import/Importer/OldImporterInterface.php diff --git a/app/Import/Setup/CsvSetup.php b/app/Import/Setup/CsvSetup.php index e3888e0030..839d2cca65 100644 --- a/app/Import/Setup/CsvSetup.php +++ b/app/Import/Setup/CsvSetup.php @@ -237,7 +237,6 @@ class CsvSetup implements SetupInterface $all = $request->all(); if ($request->get('settings') == 'roles') { $count = $config['column-count']; - $roleSet = 0; // how many roles have been defined $mapSet = 0; // how many columns must be mapped for ($i = 0; $i < $count; $i++) { diff --git a/app/Models/ImportJob.php b/app/Models/ImportJob.php index 66dfc167eb..fb1ae18c15 100644 --- a/app/Models/ImportJob.php +++ b/app/Models/ImportJob.php @@ -42,11 +42,9 @@ class ImportJob extends Model protected $validStatus = [ - 'import_status_never_started', // initial state - 'import_configuration_saved', // import configuration saved. This step is going to be obsolete. - 'settings_complete', // aka: ready for import. - 'import_running', // import currently underway - 'import_complete', // done with everything + 'new', + 'initialized', + 'configured', ]; /** diff --git a/app/Repositories/ImportJob/ImportJobRepository.php b/app/Repositories/ImportJob/ImportJobRepository.php index 6af984a06c..c1f597c346 100644 --- a/app/Repositories/ImportJob/ImportJobRepository.php +++ b/app/Repositories/ImportJob/ImportJobRepository.php @@ -13,10 +13,16 @@ declare(strict_types=1); namespace FireflyIII\Repositories\ImportJob; +use Crypt; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\User\UserRepositoryInterface; use FireflyIII\User; use Illuminate\Support\Str; +use Log; +use SplFileObject; +use Storage; +use Symfony\Component\HttpFoundation\File\UploadedFile; /** * Class ImportJobRepository @@ -51,7 +57,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface $importJob->user()->associate($this->user); $importJob->file_type = $fileType; $importJob->key = Str::random(12); - $importJob->status = 'import_status_never_started'; + $importJob->status = 'new'; $importJob->extended_status = [ 'total_steps' => 0, 'steps_done' => 0, @@ -86,6 +92,77 @@ class ImportJobRepository implements ImportJobRepositoryInterface return $result; } + /** + * @param ImportJob $job + * @param UploadedFile $file + * + * @return bool + */ + public function processConfiguration(ImportJob $job, UploadedFile $file): bool + { + /** @var UserRepositoryInterface $repository */ + $repository = app(UserRepositoryInterface::class); + // demo user's configuration upload is ignored completely. + if ($repository->hasRole($this->user, 'demo')) { + Log::debug( + 'Uploaded configuration file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()] + ); + + $configFileObject = new SplFileObject($file->getRealPath()); + $configRaw = $configFileObject->fread($configFileObject->getSize()); + $configuration = json_decode($configRaw, true); + + if (!is_null($configuration) && is_array($configuration)) { + Log::debug('Found configuration', $configuration); + $this->setConfiguration($job, $configuration); + } + } + + return true; + } + + /** + * @param ImportJob $job + * @param UploadedFile $file + * + * @return mixed + */ + public function processFile(ImportJob $job, UploadedFile $file): bool + { + /** @var UserRepositoryInterface $repository */ + $repository = app(UserRepositoryInterface::class); + $newName = sprintf('%s.upload', $job->key); + $uploaded = new SplFileObject($file->getRealPath()); + $content = $uploaded->fread($uploaded->getSize()); + $contentEncrypted = Crypt::encrypt($content); + $disk = Storage::disk('upload'); + + + // user is demo user, replace upload with prepared file. + if ($repository->hasRole($this->user, 'demo')) { + $stubsDisk = Storage::disk('stubs'); + $content = $stubsDisk->get('demo-import.csv'); + $contentEncrypted = Crypt::encrypt($content); + $disk->put($newName, $contentEncrypted); + Log::debug('Replaced upload with demo file.'); + + // also set up prepared configuration. + $configuration = json_decode($stubsDisk->get('demo-configuration.json'), true); + $this->setConfiguration($job, $configuration); + Log::debug('Set configuration for demo user', $configuration); + } + + if (!$repository->hasRole($this->user, 'demo')) { + // user is not demo, process original upload: + $disk->put($newName, $contentEncrypted); + Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]); + } + $job->status = 'initialized'; + $job->save(); + + return true; + } + /** * @param ImportJob $job * @param array $configuration diff --git a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php index 5bdf636d42..421a7849b2 100644 --- a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php +++ b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php @@ -15,6 +15,7 @@ namespace FireflyIII\Repositories\ImportJob; use FireflyIII\Models\ImportJob; use FireflyIII\User; +use Symfony\Component\HttpFoundation\File\UploadedFile; /** * Interface ImportJobRepositoryInterface @@ -37,6 +38,22 @@ interface ImportJobRepositoryInterface */ public function findByKey(string $key): ImportJob; + /** + * @param ImportJob $job + * @param UploadedFile $file + * + * @return mixed + */ + public function processFile(ImportJob $job, UploadedFile $file): bool; + + /** + * @param ImportJob $job + * @param UploadedFile $file + * + * @return bool + */ + public function processConfiguration(ImportJob $job, UploadedFile $file): bool; + /** * @param ImportJob $job * @param array $configuration diff --git a/app/Support/Import/Configuration/ConfigurationInterface.php b/app/Support/Import/Configuration/ConfigurationInterface.php new file mode 100644 index 0000000000..7cc3140782 --- /dev/null +++ b/app/Support/Import/Configuration/ConfigurationInterface.php @@ -0,0 +1,46 @@ +job = $job; + } + + /** + * @return array + */ + public function getData(): array + { + /** @var AccountRepositoryInterface $accountRepository */ + $accountRepository = app(AccountRepositoryInterface::class); + $accounts = $accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]); + $delimiters = [ + ',' => trans('form.csv_comma'), + ';' => trans('form.csv_semicolon'), + 'tab' => trans('form.csv_tab'), + ]; + + $specifics = []; + + // collect specifics. + foreach (config('csv.import_specifics') as $name => $className) { + $specifics[$name] = [ + 'name' => $className::getName(), + 'description' => $className::getDescription(), + ]; + } + + $data = [ + 'accounts' => ExpandedForm::makeSelectList($accounts), + 'specifix' => [], + 'delimiters' => $delimiters, + 'specifics' => $specifics, + ]; + + return $data; + } + + /** + * Store the result. + * + * @param array $data + * + * @return bool + */ + public function storeConfiguration(array $data): bool + { + /** @var AccountRepositoryInterface $repository */ + $repository = app(AccountRepositoryInterface::class); + $importId = $data['csv_import_account'] ?? 0; + $account = $repository->find(intval($importId)); + + $hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false; + $config = $this->job->configuration; + $config['initial-config-complete'] = true; + $config['has-headers'] = $hasHeaders; + $config['date-format'] = $data['date_format']; + $config['delimiter'] = $data['csv_delimiter']; + $config['delimiter'] = $config['delimiter'] === 'tab' ? "\t" : $config['delimiter']; + + Log::debug('Entered import account.', ['id' => $importId]); + + + if (!is_null($account->id)) { + Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]); + $config['import-account'] = $account->id; + } + if (is_null($account->id)) { + Log::error('Could not find anything for csv_import_account.', ['id' => $importId]); + } + + // loop specifics. + if (isset($data['specifics']) && is_array($data['specifics'])) { + foreach ($data['specifics'] as $name => $enabled) { + // verify their content. + $className = sprintf('FireflyIII\Import\Specifics\%s', $name); + if (class_exists($className)) { + $config['specifics'][$name] = 1; + } + } + } + $this->job->configuration = $config; + $this->job->save(); + + return true; + } +} \ No newline at end of file diff --git a/app/Support/Import/Configuration/Csv/Map.php b/app/Support/Import/Configuration/Csv/Map.php new file mode 100644 index 0000000000..f46e4d9188 --- /dev/null +++ b/app/Support/Import/Configuration/Csv/Map.php @@ -0,0 +1,229 @@ +job = $job; + } + + /** + * @return array + * @throws FireflyException + */ + public function getData(): array + { + $config = $this->job->configuration; + $this->getMappableColumns(); + + + // in order to actually map we also need all possible values from the CSV file. + $content = $this->job->uploadFileContents(); + /** @var Reader $reader */ + $reader = Reader::createFromString($content); + $reader->setDelimiter($config['delimiter']); + $results = $reader->fetch(); + $validSpecifics = array_keys(config('csv.import_specifics')); + + foreach ($results as $rowIndex => $row) { + + // skip first row? + if ($rowIndex === 0 && $config['has-headers']) { + continue; + } + + // run specifics here: + // and this is the point where the specifix go to work. + foreach ($config['specifics'] as $name => $enabled) { + + if (!in_array($name, $validSpecifics)) { + throw new FireflyException(sprintf('"%s" is not a valid class name', $name)); + } + $class = config('csv.import_specifics.' . $name); + /** @var SpecificInterface $specific */ + $specific = app($class); + + // it returns the row, possibly modified: + $row = $specific->run($row); + } + + //do something here + foreach ($indexes as $index) { // this is simply 1, 2, 3, etc. + if (!isset($row[$index])) { + // don't really know how to handle this. Just skip, for now. + continue; + } + $value = $row[$index]; + if (strlen($value) > 0) { + + // we can do some preprocessing here, + // which is exclusively to fix the tags: + if (!is_null($data[$index]['preProcessMap'])) { + /** @var PreProcessorInterface $preProcessor */ + $preProcessor = app($data[$index]['preProcessMap']); + $result = $preProcessor->run($value); + $data[$index]['values'] = array_merge($data[$index]['values'], $result); + + Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]); + Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]); + Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $data[$index]['values']]); + + + continue; + } + + $data[$index]['values'][] = $value; + } + } + } + foreach ($data as $index => $entry) { + $data[$index]['values'] = array_unique($data[$index]['values']); + } + + return $data; + } + + /** + * Store the result. + * + * @param array $data + * + * @return bool + */ + public function storeConfiguration(array $data): bool + { + return true; + } + + /** + * @param string $column + * + * @return MapperInterface + */ + private function createMapper(string $column): MapperInterface + { + $mapperClass = config('csv.import_roles.' . $column . '.mapper'); + $mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass); + /** @var MapperInterface $mapper */ + $mapper = new $mapperName; + + return $mapper; + } + + /** + * @return bool + */ + private function getMappableColumns(): bool + { + $config = $this->job->configuration; + + /** + * @var int $index + * @var bool $mustBeMapped + */ + foreach ($config['column-do-mapping'] as $index => $mustBeMapped) { + $column = $this->validateColumnName($config['column-roles'][$index] ?? '_ignore'); + $shouldMap = $this->shouldMapColumn($column, $mustBeMapped); + if ($shouldMap) { + + + // create configuration entry for this specific column and add column to $this->data array for later processing. + $this->data[$index] = [ + 'name' => $column, + 'index' => $index, + 'options' => $this->createMapper($column)->getMap(), + 'preProcessMap' => $this->getPreProcessorName($column), + 'values' => [], + ]; + } + } + + return true; + + } + + /** + * @param string $column + * + * @return string + */ + private function getPreProcessorName(string $column): string + { + $name = ''; + $hasPreProcess = config('csv.import_roles.' . $column . '.pre-process-map'); + $preProcessClass = config('csv.import_roles.' . $column . '.pre-process-mapper'); + + if (!is_null($hasPreProcess) && $hasPreProcess === true && !is_null($preProcessClass)) { + $name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass); + } + + return $name; + } + + /** + * @param string $column + * @param bool $mustBeMapped + * + * @return bool + */ + private function shouldMapColumn(string $column, bool $mustBeMapped): bool + { + $canBeMapped = config('csv.import_roles.' . $column . '.mappable'); + + return ($canBeMapped && $mustBeMapped); + } + + /** + * @param string $column + * + * @return string + * @throws FireflyException + */ + private function validateColumnName(string $column): string + { + // is valid column? + $validColumns = array_keys(config('csv.import_roles')); + if (!in_array($column, $validColumns)) { + throw new FireflyException(sprintf('"%s" is not a valid column.', $column)); + } + + return $column; + } +} \ No newline at end of file diff --git a/app/Support/Import/Configuration/Csv/Roles.php b/app/Support/Import/Configuration/Csv/Roles.php new file mode 100644 index 0000000000..d439af415e --- /dev/null +++ b/app/Support/Import/Configuration/Csv/Roles.php @@ -0,0 +1,261 @@ +job = $job; + } + + /** + * Get the data necessary to show the configuration screen. + * + * @return array + */ + public function getData(): array + { + $config = $this->job->configuration; + $content = $this->job->uploadFileContents(); + + // create CSV reader. + $reader = Reader::createFromString($content); + $reader->setDelimiter($config['delimiter']); + $start = $config['has-headers'] ? 1 : 0; + $end = $start + config('csv.example_rows'); + + // set data: + $this->data = [ + 'examples' => [], + 'roles' => $this->getRoles(), + 'total' => 0, + 'headers' => $config['has-headers'] ? $reader->fetchOne(0) : [], + ]; + + while ($start < $end) { + $row = $reader->fetchOne($start); + $row = $this->processSpecifics($row); + $count = count($row); + $this->data['total'] = $count > $this->data['total'] ? $count : $this->data['total']; + $this->processRow($row); + $start++; + } + + $this->updateColumCount(); + $this->makeExamplesUnique(); + + return $this->data; + } + + /** + * Store the result. + * + * @param array $data + * + * @return bool + */ + public function storeConfiguration(array $data): bool + { + Log::debug('Now in storeConfiguration of Roles.'); + $config = $this->job->configuration; + $count = $config['column-count']; + for ($i = 0; $i < $count; $i++) { + $role = $data['role'][$i] ?? '_ignore'; + $mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false; + $config['column-roles'][$i] = $role; + $config['column-do-mapping'][$i] = $mapping; + Log::debug(sprintf('Column %d has been given role %s', $i, $role)); + } + + + $this->job->configuration = $config; + $this->job->save(); + + $this->ignoreUnmappableColumns(); + $this->setRolesComplete(); + $this->isMappingNecessary(); + + + return true; + } + + /** + * @return array + */ + private function getRoles(): array + { + $roles = []; + foreach (array_keys(config('csv.import_roles')) as $role) { + $roles[$role] = trans('csv.column_' . $role); + } + + return $roles; + + } + + /** + * @return bool + */ + private function ignoreUnmappableColumns(): bool + { + $config = $this->job->configuration; + $count = $config['column-count']; + for ($i = 0; $i < $count; $i++) { + $role = $config['column-roles'][$i] ?? '_ignore'; + $mapping = $config['column-do-mapping'][$i] ?? false; + + if ($role === '_ignore' && $mapping === true) { + $mapping = false; + Log::debug(sprintf('Column %d has type %s so it cannot be mapped.', $i, $role)); + } + $config['column-do-mapping'][$i] = $mapping; + } + + $this->job->configuration = $config; + $this->job->save(); + + return true; + + } + + /** + * @return bool + */ + private function isMappingNecessary() + { + $config = $this->job->configuration; + $count = $config['column-count']; + $toBeMapped = 0; + for ($i = 0; $i < $count; $i++) { + $mapping = $config['column-do-mapping'][$i] ?? false; + if ($mapping === true) { + $toBeMapped++; + } + } + Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped)); + if ($toBeMapped === 0) { + // skip setting of map, because none need to be mapped: + $config['column-mapping-complete'] = true; + $this->job->configuration = $config; + $this->job->save(); + } + + return true; + } + + /** + * make unique example data + */ + private function makeExamplesUnique(): bool + { + foreach ($this->data['examples'] as $index => $values) { + $this->data['examples'][$index] = array_unique($values); + } + + return true; + } + + /** + * @param array $row + * + * @return bool + */ + private function processRow(array $row): bool + { + foreach ($row as $index => $value) { + $value = trim($value); + if (strlen($value) > 0) { + $this->data['examples'][$index][] = $value; + } + } + + return true; + } + + /** + * run specifics here: + * and this is the point where the specifix go to work. + * + * @param array $row + * + * @return array + */ + private function processSpecifics(array $row): array + { + foreach ($this->job->configuration['specifics'] as $name => $enabled) { + /** @var SpecificInterface $specific */ + $specific = app('FireflyIII\Import\Specifics\\' . $name); + $row = $specific->run($row); + } + + return $row; + } + + /** + * @return bool + */ + private function setRolesComplete(): bool + { + $config = $this->job->configuration; + $count = $config['column-count']; + $assigned = 0; + for ($i = 0; $i < $count; $i++) { + $role = $config['column-roles'][$i] ?? '_ignore'; + if ($role !== '_ignore') { + $assigned++; + } + } + if ($assigned > 0) { + $config['column-roles-complete'] = true; + $this->job->configuration = $config; + $this->job->save(); + } + + return true; + } + + /** + * @return bool + */ + private function updateColumCount(): bool + { + $config = $this->job->configuration; + $count = $this->data['total']; + $config['column-count'] = $count; + $this->job->configuration = $config; + $this->job->save(); + + return true; + } +} \ No newline at end of file diff --git a/config/csv.php b/config/csv.php index 5da40e41e0..cc23de32f1 100644 --- a/config/csv.php +++ b/config/csv.php @@ -292,6 +292,7 @@ return [ // number of example rows: 'example_rows' => 5, 'default_config' => [ + 'initial-config-complete' => false, 'has-headers' => false, // assume 'date-format' => 'Ymd', // assume 'delimiter' => ',', // assume diff --git a/config/firefly.php b/config/firefly.php index c4f648e3a4..e00c6d32d4 100644 --- a/config/firefly.php +++ b/config/firefly.php @@ -31,7 +31,10 @@ return [ 'csv' => 'FireflyIII\Export\Exporter\CsvExporter', ], 'import_formats' => [ - 'csv' => 'FireflyIII\Import\Importer\CsvImporter', + 'csv' => 'FireflyIII\Import\Configurator\CsvConfigurator', + ], + 'import_configurators' => [ + 'csv' => 'FireflyIII\Import\Configurator\CsvConfigurator', ], 'default_export_format' => 'csv', 'default_import_format' => 'csv', diff --git a/public/js/ff/import/status.js b/public/js/ff/import/status.js index cc1ec69a9e..75671c2971 100644 --- a/public/js/ff/import/status.js +++ b/public/js/ff/import/status.js @@ -10,6 +10,10 @@ /** global: jobImportUrl, langImportSingleError, langImportMultiError, jobStartUrl, langImportTimeOutError, langImportFinished, langImportFatalError */ +var displayStatus = 'initial'; +var timeOutId; + + var startedImport = false; var startInterval = 2000; var interval = 500; @@ -19,20 +23,85 @@ var stepCount = 0; $(function () { "use strict"; - $('#import-status-intro').hide(); - $('#import-status-more-info').hide(); + //$('#import-status-intro').hide(); + //$('#import-status-more-info').hide(); // check status, every 500 ms. - setTimeout(checkImportStatus, startInterval); + timeOutId = setTimeout(checkImportStatus, startInterval); + + // button to start import routine: + $('.start-job').click(startJob); }); +function startJob() { + console.log('Job started.'); + $.post(jobStartUrl); + return false; +} function checkImportStatus() { "use strict"; $.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport); } +function reportOnJobImport(data) { + "use strict"; + displayCorrectBox(data.status); + //updateBar(data); + //reportErrors(data); + //reportStatus(data); + //updateTimeout(data); + + //if (importJobFinished(data)) { + // finishedJob(data); + // return; + //} + + + // same number of steps as last time? + //if (currentLimit > timeoutLimit) { + // timeoutError(); + // return; + //} + + // if the job has not actually started, do so now: + //if (!data.started && !startedImport) { + // kickStartJob(); + // return; + //} + + // trigger another check. + //timeOutId = setTimeout(checkImportStatus, interval); + +} + +function displayCorrectBox(status) { + console.log('Current job state is ' + status); + if(status === 'configured' && displayStatus === 'initial') { + // hide some boxes: + $('.status_initial').hide(); + return; + } + console.error('CANNOT HANDLE CURRENT STATE'); +} + + + + + + + + + + + + + + + + + function importComplete() { "use strict"; var bar = $('#import-status-bar'); @@ -131,35 +200,7 @@ function finishedJob(data) { } -function reportOnJobImport(data) { - "use strict"; - updateBar(data); - reportErrors(data); - reportStatus(data); - updateTimeout(data); - if (importJobFinished(data)) { - finishedJob(data); - return; - } - - - // same number of steps as last time? - if (currentLimit > timeoutLimit) { - timeoutError(); - return; - } - - // if the job has not actually started, do so now: - if (!data.started && !startedImport) { - kickStartJob(); - return; - } - - // trigger another check. - setTimeout(checkImportStatus, interval); - -} function startedTheImport() { "use strict"; diff --git a/resources/lang/en_US/csv.php b/resources/lang/en_US/csv.php index d5306e2a88..837616a801 100644 --- a/resources/lang/en_US/csv.php +++ b/resources/lang/en_US/csv.php @@ -13,28 +13,32 @@ declare(strict_types=1); return [ - 'import_configure_title' => 'Configure your import', - 'import_configure_intro' => 'There are some options for your CSV import. Please indicate if your CSV file contains headers on the first column, and what the date format of your date-fields is. That might require some experimentation. The field delimiter is usually a ",", but could also be a ";". Check this carefully.', - 'import_configure_form' => 'Basic CSV import options', - 'header_help' => 'Check this if the first row of your CSV file are the column titles', - 'date_help' => 'Date time format in your CSV. Follow the format like this page indicates. The default value will parse dates that look like this: :dateExample.', - 'delimiter_help' => 'Choose the field delimiter that is used in your input file. If not sure, comma is the safest option.', - 'import_account_help' => 'If your CSV file does NOT contain information about your asset account(s), use this dropdown to select to which account the transactions in the CSV belong to.', - 'upload_not_writeable' => 'The grey box contains a file path. It should be writeable. Please make sure it is.', + // initial config + 'initial_config_title' => 'Import configuration (1/3)', + 'initial_config_text' => 'To be able to import your file correctly, please validate the options below.', + 'initial_config_box' => 'Basic CSV import configuration', + 'initial_header_help' => 'Check this box if the first row of your CSV file are the column titles.', + 'initial_date_help' => 'Date time format in your CSV. Follow the format like this page indicates. The default value will parse dates that look like this: :dateExample.', + 'initial_delimiter_help' => 'Choose the field delimiter that is used in your input file. If not sure, comma is the safest option.', + 'initial_import_account_help' => 'If your CSV file does NOT contain information about your asset account(s), use this dropdown to select to which account the transactions in the CSV belong to.', - // roles - 'column_roles_title' => 'Define column roles', - 'column_roles_table' => 'Table', - 'column_name' => 'Name of column', - 'column_example' => 'Column example data', - 'column_role' => 'Column data meaning', - 'do_map_value' => 'Map these values', - 'column' => 'Column', - 'no_example_data' => 'No example data available', - 'store_column_roles' => 'Continue import', - 'do_not_map' => '(do not map)', - 'map_title' => 'Connect import data to Firefly III data', - 'map_text' => 'In the following tables, the left value shows you information found in your uploaded CSV file. It is your task to map this value, if possible, to a value already present in your database. Firefly will stick to this mapping. If there is no value to map to, or you do not wish to map the specific value, select nothing.', + // roles config + 'roles_title' => 'Define each column\'s role', + 'roles_text' => 'Each column in your CSV file contains certain data. Please indicate what kind of data the importer should expect. The option to "map" data means that you will link each entry found in the column to a value in your database. An often mapped column is the column that contains the IBAN of the opposing account. That can be easily matched to IBAN\'s present in your database already.', + 'roles_table' => 'Table', + 'roles_column_name' => 'Name of column', + 'roles_column_example' => 'Column example data', + 'roles_column_role' => 'Column data meaning', + 'roles_do_map_value' => 'Map these values', + 'roles_column' => 'Column', + 'roles_no_example_data' => 'No example data available', + + 'roles_store' => 'Continue import', + 'roles_do_not_map' => '(do not map)', + + // map data + 'map_title' => 'Connect import data to Firefly III data', + 'map_text' => 'In the following tables, the left value shows you information found in your uploaded CSV file. It is your task to map this value, if possible, to a value already present in your database. Firefly will stick to this mapping. If there is no value to map to, or you do not wish to map the specific value, select nothing.', 'field_value' => 'Field value', 'field_mapped_to' => 'Mapped to', diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index 501c03430a..2755ae6f58 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -1003,7 +1003,6 @@ return [ 'import_finished_report' => 'The import has finished. Please note any errors in the block above this line. All transactions imported during this particular session have been tagged, and you can check them out below. ', 'import_finished_link' => 'The transactions imported can be found in tag :tag.', 'need_at_least_one_account' => 'You need at least one asset account to be able to create piggy banks', - 'see_help_top_right' => 'For more information, please check out the help pages using the icon in the top right corner of the page.', 'bread_crumb_import_complete' => 'Import ":key" complete', 'bread_crumb_configure_import' => 'Configure import ":key"', 'bread_crumb_import_finished' => 'Import ":key" finished', diff --git a/resources/views/import/csv/configure.twig b/resources/views/import/csv/initial.twig similarity index 61% rename from resources/views/import/csv/configure.twig rename to resources/views/import/csv/initial.twig index 9a9e53a0b0..2af4c072b8 100644 --- a/resources/views/import/csv/configure.twig +++ b/resources/views/import/csv/initial.twig @@ -10,11 +10,11 @@
-

{{ trans('csv.import_configure_title') }}

+

{{ trans('csv.initial_config_title') }}

- {{ trans('csv.import_configure_intro') }} + {{ trans('csv.initial_config_text') }}

@@ -29,16 +29,16 @@
-

{{ trans('csv.import_configure_form') }}

+

{{ trans('csv.initial_config_box') }}

- {{ ExpandedForm.checkbox('has_headers',1,job.configuration['has-headers'],{helpText: trans('csv.header_help')}) }} - {{ ExpandedForm.text('date_format',job.configuration['date-format'],{helpText: trans('csv.date_help', {dateExample: phpdate('Ymd')}) }) }} - {{ ExpandedForm.select('csv_delimiter', data.delimiters, job.configuration['delimiter'], {helpText: trans('csv.delimiter_help') } ) }} - {{ ExpandedForm.select('csv_import_account', data.accounts, job.configuration['import-account'], {helpText: trans('csv.import_account_help')} ) }} + {{ ExpandedForm.checkbox('has_headers',1,job.configuration['has-headers'],{helpText: trans('csv.initial_header_help')}) }} + {{ ExpandedForm.text('date_format',job.configuration['date-format'],{helpText: trans('csv.initial_date_help', {dateExample: phpdate('Ymd')}) }) }} + {{ ExpandedForm.select('csv_delimiter', data.delimiters, job.configuration['delimiter'], {helpText: trans('csv.initial_delimiter_help') } ) }} + {{ ExpandedForm.select('csv_import_account', data.accounts, job.configuration['import-account'], {helpText: trans('csv.initial_import_account_help')} ) }} {% for type, specific in data.specifics %}
@@ -56,40 +56,23 @@
{% endfor %} - - {% if not data.is_upload_possible %} -
-
-   -
- -
- 
{{ data.upload_path }}
-

- {{ trans('csv.upload_not_writeable') }} -

-
-
- {% endif %}
- {% if data.is_upload_possible %} -
-
-
-
- -
+
+
+
+
+
- {% endif %} +
diff --git a/resources/views/import/csv/map.twig b/resources/views/import/csv/map.twig index ebc0cf669d..3fd6f31d57 100644 --- a/resources/views/import/csv/map.twig +++ b/resources/views/import/csv/map.twig @@ -1,7 +1,7 @@ {% extends "./layout/default" %} {% block breadcrumbs %} - {{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }} + {{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName, job) }} {% endblock %} {% block content %} @@ -22,9 +22,8 @@
-
+ - {% for field in data %}
diff --git a/resources/views/import/csv/roles.twig b/resources/views/import/csv/roles.twig index 0df197187b..307498f52c 100644 --- a/resources/views/import/csv/roles.twig +++ b/resources/views/import/csv/roles.twig @@ -1,7 +1,7 @@ {% extends "./layout/default" %} {% block breadcrumbs %} - {{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }} + {{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName, job) }} {% endblock %} {% block content %} @@ -10,18 +10,18 @@
-

{{ trans('csv.column_roles_title') }}

+

{{ trans('csv.roles_title') }}

- {{ 'see_help_top_right'|_ }} + {{ trans('csv.roles_text') }}

- + @@ -29,41 +29,41 @@
-

{{ trans('csv.column_roles_table') }}

+

{{ trans('csv.roles_table') }}

- - - - + + + + - {% for i in 0..(data.columnCount-1) %} + {% for i in 0..(data.total -1) %} @@ -91,7 +91,7 @@
diff --git a/resources/views/import/index.twig b/resources/views/import/index.twig index 922a3fa12e..2f8ff89345 100644 --- a/resources/views/import/index.twig +++ b/resources/views/import/index.twig @@ -23,20 +23,19 @@
-
+ {{ ExpandedForm.file('import_file', {helpText: 'import_file_help'|_}) }} {{ ExpandedForm.file('configuration_file', {helpText: 'configuration_file_help'|_|raw}) }} - {{ ExpandedForm.select('import_file_type', importFileTypes, defaultImportType, {'helpText' : 'import_file_type_help'|_}) }}
-
{{ trans('csv.column_name') }}{{ trans('csv.column_example') }}{{ trans('csv.column_role') }}{{ trans('csv.do_map_value') }}{{ trans('csv.roles_column_name') }}{{ trans('csv.roles_column_example') }}{{ trans('csv.roles_column_role') }}{{ trans('csv.roles_do_map_value') }}
- {% if data.columnHeaders[i] == '' %} - {{ trans('csv.column') }} #{{ loop.index }} + {% if data.headers[i] == '' %} + {{ trans('csv.roles_column') }} #{{ loop.index }} {% else %} - {{ data.columnHeaders[i] }} + {{ data.headers[i] }} {% endif %} - {% if data.columns[i]|length == 0 %} - {{ trans('csv.no_example_data') }} + {% if data.examples[i]|length == 0 %} + {{ trans('csv.roles_no_example_data') }} {% else %} - {% for example in data.columns[i] %} + {% for example in data.examples[i] %} {{ example }}
{% endfor %} {% endif %} 		{{ Form.select(('role['~loop.index0~']'), - data.available_roles, + data.roles, job.configuration['column-roles'][loop.index0], {class: 'form-control'}) }}