From 18d274181407246ccf25d9def481e4e35face8e8 Mon Sep 17 00:00:00 2001 From: James Cole Date: Mon, 27 Jun 2016 15:15:46 +0200 Subject: [PATCH] More code for the new CSV import --- app/Http/Controllers/ImportController.php | 18 +- app/Http/routes.php | 3 +- app/Import/Importer/CsvImporter.php | 93 +++++++++- app/Import/Importer/ImporterInterface.php | 30 +++- app/Models/ImportJob.php | 36 ++-- config/csv.php | 203 ++++++++++++++++++++++ config/firefly.php | 18 +- resources/lang/en_US/csv.php | 44 +++++ resources/views/import/csv/map.twig | 88 ++++++++++ 9 files changed, 499 insertions(+), 34 deletions(-) create mode 100644 resources/lang/en_US/csv.php create mode 100644 resources/views/import/csv/map.twig diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index 4bd6173903..3416a5a6fd 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -85,7 +85,7 @@ class ImportController extends Controller * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector * @throws FireflyException */ - public function process(Request $request, ImportJob $job) + public function postConfigure(Request $request, ImportJob $job) { if (!$this->jobInCorrectStep($job, 'process')) { return $this->redirectToCorrectStep($job); @@ -94,7 +94,8 @@ class ImportController extends Controller // actual code $importer = $this->makeImporter($job); $data = $request->all(); - $importer->saveImportConfiguration($data); + $files = $request->files; + $importer->saveImportConfiguration($data, $files); // update job: $job->status = 'import_configuration_saved'; @@ -121,11 +122,18 @@ class ImportController extends Controller } $importer = $this->makeImporter($job); - // now + // now show settings screen to user. + if ($importer->requireUserSettings()) { + $data = $importer->getDataForSettings(); + $view = $importer->getViewForSettings(); + + return view($view, compact('data', 'job')); + } + + // if no more settings, save job and continue to process thing. - - echo 'now in settings'; + echo 'now in settings (done)'; exit; // actual code diff --git a/app/Http/routes.php b/app/Http/routes.php index 29f99c3f4f..662fafce7e 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -226,8 +226,9 @@ Route::group( Route::get('/import', ['uses' => 'ImportController@index', 'as' => 'import.index']); Route::post('/import/upload', ['uses' => 'ImportController@upload', 'as' => 'import.upload']); Route::get('/import/configure/{importJob}', ['uses' => 'ImportController@configure', 'as' => 'import.configure']); - Route::post('/import/process/{importJob}', ['uses' => 'ImportController@process', 'as' => 'import.process_configuration']); + Route::post('/import/configure/{importJob}', ['uses' => 'ImportController@postConfigure', 'as' => 'import.process_configuration']); Route::get('/import/settings/{importJob}', ['uses' => 'ImportController@settings', 'as' => 'import.settings']); + Route::post('/import/settings/{importJob}', ['uses' => 'ImportController@postSettings', 'as' => 'import.postSettings']); /** diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/CsvImporter.php index 82b83e1dc8..ece4c9e527 100644 --- a/app/Import/Importer/CsvImporter.php +++ b/app/Import/Importer/CsvImporter.php @@ -17,6 +17,8 @@ use FireflyIII\Crud\Account\AccountCrud; use FireflyIII\Import\Role\Map; use FireflyIII\Models\AccountType; use FireflyIII\Models\ImportJob; +use League\Csv\Reader; +use Symfony\Component\HttpFoundation\FileBag; /** * Class CsvImporter @@ -25,6 +27,7 @@ use FireflyIII\Models\ImportJob; */ class CsvImporter implements ImporterInterface { + const EXAMPLE_ROWS = 5; /** @var ImportJob */ public $job; @@ -54,7 +57,7 @@ class CsvImporter implements ImporterInterface $specifics = []; // collect specifics. - foreach (config('firefly.csv_import_specifics') as $name => $className) { + foreach (config('csv.import_specifics') as $name => $className) { $specifics[$name] = [ 'name' => $className::getName(), 'description' => $className::getDescription(), @@ -73,6 +76,68 @@ class CsvImporter implements ImporterInterface return $data; } + /** + * This method returns the data required for the view that will let the user add settings to the import job. + * + * @return array + */ + public function getDataForSettings(): array + { + $config = $this->job->configuration; + $data = [ + 'columns' => [], + 'columnCount' => 0, + ]; + + if (!isset($config['columns'])) { + + // show user column configuration. + $content = $this->job->uploadFileContents(); + + // create CSV reader. + $reader = Reader::createFromString($content); + $start = $config['has_headers'] ? 1 : 0; + $end = $start + self::EXAMPLE_ROWS; // first X rows + while ($start < $end) { + $row = $reader->fetchOne($start); + foreach ($row as $index => $value) { + $value = trim($value); + if (strlen($value) > 0) { + $data['columns'][$index][] = $value; + } + } + $start++; + $data['columnCount'] = count($row); + } + + // make unique + foreach ($data['columns'] as $index => $values) { + $data['columns'][$index] = array_unique($values); + } + // TODO preset roles from config + $data['set_roles'] = []; + // collect possible column roles: + $data['available_roles'] = []; + foreach (array_keys(config('csv.import_roles')) as $role) { + $data['available_roles'][$role] = trans('csv.csv_column_'.$role); + } + + return $data; + } + + } + + /** + * This method returns the name of the view that will be shown to the user to further configure + * the import job. + * + * @return string + */ + public function getViewForSettings(): string + { + return 'import.csv.map'; + } + /** * Returns a Map thing used to allow the user to * define roles for each entry. @@ -85,21 +150,45 @@ class CsvImporter implements ImporterInterface exit; } + /** + * This method returns whether or not the user must configure this import + * job further. + * + * @return bool + */ + public function requireUserSettings(): bool + { + // does the job have both a 'map' array and a 'columns' array. + $config = $this->job->configuration; + if (isset($config['map']) && isset($config['columns'])) { + return false; + } + + return true; + } + /** * @param array $data * * @return bool */ - public function saveImportConfiguration(array $data): bool + public function saveImportConfiguration(array $data, FileBag $files): bool { + /* + * TODO file upload is ignored for now. + */ + /** @var AccountCrud $repository */ $repository = app(AccountCrud::class); $account = $repository->find(intval($data['csv_import_account'])); + $hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false; $configuration = [ + 'has_headers' => $hasHeaders, 'date_format' => $data['date_format'], 'csv_delimiter' => $data['csv_delimiter'], 'csv_import_account' => 0, 'specifics' => [], + ]; if (!is_null($account->id)) { diff --git a/app/Import/Importer/ImporterInterface.php b/app/Import/Importer/ImporterInterface.php index 8116f252a7..45135fc26b 100644 --- a/app/Import/Importer/ImporterInterface.php +++ b/app/Import/Importer/ImporterInterface.php @@ -13,6 +13,7 @@ namespace FireflyIII\Import\Importer; use FireflyIII\Import\Role\Map; use FireflyIII\Models\ImportJob; +use Symfony\Component\HttpFoundation\FileBag; /** * Interface ImporterInterface @@ -37,11 +38,19 @@ interface ImporterInterface public function getConfigurationData(): array; /** - * @param array $data + * This method returns the data required for the view that will let the user add settings to the import job. * - * @return bool + * @return array */ - public function saveImportConfiguration(array $data): bool; + public function getDataForSettings(): array; + + /** + * This method returns the name of the view that will be shown to the user to further configure + * the import job. + * + * @return string + */ + public function getViewForSettings(): string; /** * Returns a Map thing used to allow the user to @@ -51,6 +60,21 @@ interface ImporterInterface */ public function prepareRoles(): Map; + /** + * This method returns whether or not the user must configure this import + * job further. + * + * @return bool + */ + public function requireUserSettings(): bool; + + /** + * @param array $data + * + * @return bool + */ + public function saveImportConfiguration(array $data, FileBag $files): bool; + /** * @param ImportJob $job * diff --git a/app/Models/ImportJob.php b/app/Models/ImportJob.php index 6335d78c10..1f679e0d70 100644 --- a/app/Models/ImportJob.php +++ b/app/Models/ImportJob.php @@ -12,7 +12,9 @@ declare(strict_types = 1); namespace FireflyIII\Models; use Auth; +use Crypt; use Illuminate\Database\Eloquent\Model; +use Storage; use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; @@ -67,15 +69,6 @@ class ImportJob extends Model $this->save(); } - /** - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo - */ - public function user() - { - return $this->belongsTo('FireflyIII\User'); - } - - /** * @param $value * @@ -86,8 +79,8 @@ class ImportJob extends Model if (strlen($value) == 0) { return []; } - - return json_decode($value); + + return json_decode($value, true); } /** @@ -97,4 +90,25 @@ class ImportJob extends Model { $this->attributes['configuration'] = json_encode($value); } + + /** + * @return string + */ + public function uploadFileContents(): string + { + $fileName = $this->key . '.upload'; + $disk = Storage::disk('upload'); + $encryptedContent = $disk->get($fileName); + $content = Crypt::decrypt($encryptedContent); + + return $content; + } + + /** + * @return \Illuminate\Database\Eloquent\Relations\BelongsTo + */ + public function user() + { + return $this->belongsTo('FireflyIII\User'); + } } diff --git a/config/csv.php b/config/csv.php index 0419338367..3622624496 100644 --- a/config/csv.php +++ b/config/csv.php @@ -3,6 +3,206 @@ declare(strict_types = 1); return [ + + /* + * Configuration for the CSV specifics. + */ + 'import_specifics' => [ + 'RabobankDescription' => 'FireflyIII\Import\Specifics\RabobankDescription', + 'AbnAmroDescription' => 'FireflyIII\Import\Specifics\AbnAmroDescription', + ], + + /* + * Configuration for possible column roles. + */ + 'import_roles' => [ + '_ignore' => [ + 'mappable' => false, + 'converter' => 'Ignore', + 'field' => 'ignored', + ], + 'bill-id' => [ + 'mappable' => false, + 'field' => 'bill', + 'converter' => 'BillId', + 'mapper' => 'Bill', + ], + 'bill-name' => [ + 'mappable' => true, + 'converter' => 'BillName', + 'field' => 'bill', + 'mapper' => 'Bill', + ], + 'currency-id' => [ + 'mappable' => true, + 'converter' => 'CurrencyId', + 'field' => 'currency', + 'mapper' => 'TransactionCurrency' + ], + 'currency-name' => [ + 'mappable' => true, + 'converter' => 'CurrencyName', + 'field' => 'currency', + 'mapper' => 'TransactionCurrency' + ], + 'currency-code' => [ + 'mappable' => true, + 'converter' => 'CurrencyCode', + 'field' => 'currency', + 'mapper' => 'TransactionCurrency' + ], + 'currency-symbol' => [ + 'mappable' => true, + 'converter' => 'CurrencySymbol', + 'field' => 'currency', + 'mapper' => 'TransactionCurrency' + ], + 'description' => [ + 'mappable' => false, + 'converter' => 'Description', + 'field' => 'description', + ], + 'date-transaction' => [ + 'mappable' => false, + 'converter' => 'Date', + 'field' => 'date', + ], + 'date-rent' => [ + 'mappable' => false, + 'converter' => 'Date', + 'field' => 'date-rent', + ], + 'budget-id' => [ + 'mappable' => true, + 'converter' => 'BudgetId', + 'field' => 'budget', + 'mapper' => 'Budget', + ], + 'budget-name' => [ + 'mappable' => true, + 'converter' => 'BudgetName', + 'field' => 'budget', + 'mapper' => 'Budget', + ], + 'rabo-debet-credit' => [ + 'mappable' => false, + 'converter' => 'RabobankDebetCredit', + 'field' => 'amount-modifier', + ], + 'ing-debet-credit' => [ + 'mappable' => false, + 'converter' => 'INGDebetCredit', + 'field' => 'amount-modifier', + ], + 'category-id' => [ + 'mappable' => true, + 'converter' => 'CategoryId', + 'field' => 'category', + 'mapper' => 'Category', + ], + 'category-name' => [ + 'mappable' => true, + 'converter' => 'CategoryName', + 'field' => 'category', + 'mapper' => 'Category', + ], + 'tags-comma' => [ + 'mappable' => true, + 'field' => 'tags', + 'converter' => 'TagsComma', + 'mapper' => 'Tag', + ], + 'tags-space' => [ + 'mappable' => true, + 'field' => 'tags', + 'converter' => 'TagsSpace', + 'mapper' => 'Tag', + ], + 'account-id' => [ + 'mappable' => true, + 'mapper' => 'AssetAccount', + 'field' => 'asset-account-id', + 'converter' => 'AccountId' + ], + 'account-name' => [ + 'mappable' => true, + 'mapper' => 'AssetAccount', + 'field' => 'asset-account-name', + 'converter' => 'AssetAccountName' + ], + 'account-iban' => [ + 'mappable' => true, + 'converter' => 'AssetAccountIban', + 'field' => 'asset-account-iban', + 'mapper' => 'AssetAccount' + ], + 'account-number' => [ + 'mappable' => true, + 'converter' => 'AssetAccountNumber', + 'field' => 'asset-account-number', + 'mapper' => 'AssetAccount' + ], + 'opposing-id' => [ + 'mappable' => true, + 'field' => 'opposing-account-id', + 'converter' => 'OpposingAccountId', + 'mapper' => 'AnyAccount', + ], + 'opposing-name' => [ + 'mappable' => true, + 'field' => 'opposing-account-name', + 'converter' => 'OpposingAccountName', + 'mapper' => 'AnyAccount', + ], + 'opposing-iban' => [ + 'mappable' => true, + 'field' => 'opposing-account-iban', + 'converter' => 'OpposingAccountIban', + 'mapper' => 'AnyAccount', + ], + 'opposing-number' => [ + 'mappable' => true, + 'field' => 'opposing-account-number', + 'converter' => 'OpposingAccountNumber', + 'mapper' => 'AnyAccount', + ], + 'amount' => [ + 'mappable' => false, + 'converter' => 'Amount', + 'field' => 'amount', + ], + 'amount-comma-separated' => [ + 'mappable' => false, + 'converter' => 'AmountComma', + 'field' => 'amount', + ], + 'sepa-ct-id' => [ + 'mappable' => false, + 'converter' => 'Description', + 'field' => 'description', + ], + 'sepa-ct-op' => [ + 'mappable' => false, + 'converter' => 'Description', + 'field' => 'description', + ], + 'sepa-db' => [ + 'mappable' => false, + 'converter' => 'Description', + 'field' => 'description', + ], + ], + + + + + + + + + /* + + 'specifix' => [ 'RabobankDescription', 'AbnAmroDescription', @@ -194,4 +394,7 @@ return [ 'field' => 'description', ], ] + + + */ ]; diff --git a/config/firefly.php b/config/firefly.php index 0e93925850..f764212203 100644 --- a/config/firefly.php +++ b/config/firefly.php @@ -11,20 +11,14 @@ return [ 'resend_confirmation' => 3600, 'confirmation_age' => 14400, // four hours - 'export_formats' => [ + 'export_formats' => [ 'csv' => 'FireflyIII\Export\Exporter\CsvExporter', // mt940 FireflyIII Export Exporter MtExporter ], - 'csv_import_specifics' => [ - 'RabobankDescription' => 'FireflyIII\Import\Specifics\RabobankDescription', - 'AbnAmroDescription' => 'FireflyIII\Import\Specifics\AbnAmroDescription', - ], - 'import_formats' => [ + 'import_formats' => [ 'csv' => 'FireflyIII\Import\Importer\CsvImporter', // mt940 FireflyIII Import Importer MtImporter ], - - 'default_export_format' => 'csv', 'default_import_format' => 'csv', 'bill_periods' => ['weekly', 'monthly', 'quarterly', 'half-year', 'yearly'], @@ -142,7 +136,7 @@ return [ 'end_date' => 'FireflyIII\Support\Binder\Date', ], - 'rule-triggers' => [ + 'rule-triggers' => [ 'user_action' => 'FireflyIII\Rules\Triggers\UserAction', 'from_account_starts' => 'FireflyIII\Rules\Triggers\FromAccountStarts', 'from_account_ends' => 'FireflyIII\Rules\Triggers\FromAccountEnds', @@ -161,7 +155,7 @@ return [ 'description_contains' => 'FireflyIII\Rules\Triggers\DescriptionContains', 'description_is' => 'FireflyIII\Rules\Triggers\DescriptionIs', ], - 'rule-actions' => [ + 'rule-actions' => [ 'set_category' => 'FireflyIII\Rules\Actions\SetCategory', 'clear_category' => 'FireflyIII\Rules\Actions\ClearCategory', 'set_budget' => 'FireflyIII\Rules\Actions\SetBudget', @@ -174,7 +168,7 @@ return [ 'prepend_description' => 'FireflyIII\Rules\Actions\PrependDescription', ], // all rule actions that require text input: - 'rule-actions-text' => [ + 'rule-actions-text' => [ 'set_category', 'set_budget', 'add_tag', @@ -183,7 +177,7 @@ return [ 'append_description', 'prepend_description', ], - 'test-triggers' => [ + 'test-triggers' => [ // The maximum number of transactions shown when testing a list of triggers 'limit' => 10, diff --git a/resources/lang/en_US/csv.php b/resources/lang/en_US/csv.php new file mode 100644 index 0000000000..f9878e6938 --- /dev/null +++ b/resources/lang/en_US/csv.php @@ -0,0 +1,44 @@ + '(ignore this column)', + 'csv_column_account-iban' => 'Asset account (IBAN)', + 'csv_column_account-id' => 'Asset account ID (matching Firefly)', + 'csv_column_account-name' => 'Asset account (name)', + 'csv_column_amount' => 'Amount', + 'csv_column_amount-comma-separated' => 'Amount (comma as decimal separator)', + 'csv_column_bill-id' => 'Bill ID (matching Firefly)', + 'csv_column_bill-name' => 'Bill name', + 'csv_column_budget-id' => 'Budget ID (matching Firefly)', + 'csv_column_budget-name' => 'Budget name', + 'csv_column_category-id' => 'Category ID (matching Firefly)', + 'csv_column_category-name' => 'Category name', + 'csv_column_currency-code' => 'Currency code (ISO 4217)', + 'csv_column_currency-id' => 'Currency ID (matching Firefly)', + 'csv_column_currency-name' => 'Currency name (matching Firefly)', + 'csv_column_currency-symbol' => 'Currency symbol (matching Firefly)', + 'csv_column_date-rent' => 'Rent calculation date', + 'csv_column_date-transaction' => 'Date', + 'csv_column_description' => 'Description', + 'csv_column_opposing-iban' => 'Opposing account (IBAN)', + 'csv_column_opposing-id' => 'Opposing account ID (matching Firefly)', + 'csv_column_opposing-name' => 'Opposing account (name)', + 'csv_column_rabo-debet-credit' => 'Rabobank specific debet/credit indicator', + 'csv_column_ing-debet-credit' => 'ING specific debet/credit indicator', + 'csv_column_sepa-ct-id' => 'SEPA Credit Transfer end-to-end ID', + 'csv_column_sepa-ct-op' => 'SEPA Credit Transfer opposing account', + 'csv_column_sepa-db' => 'SEPA Direct Debet', + 'csv_column_tags-comma' => 'Tags (comma separated)', + 'csv_column_tags-space' => 'Tags (space separated)', + 'csv_column_account-number' => 'Asset account (account number)', + 'csv_column_opposing-number' => 'Opposing account (account number)', +]; \ No newline at end of file diff --git a/resources/views/import/csv/map.twig b/resources/views/import/csv/map.twig new file mode 100644 index 0000000000..fb141d4432 --- /dev/null +++ b/resources/views/import/csv/map.twig @@ -0,0 +1,88 @@ +{% extends "./layout/default.twig" %} + +{% block breadcrumbs %} + {{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }} +{% endblock %} + +{% block content %} + +
+
+
+
+

{{ 'csv_column_roles_title'|_ }}

+
+
+

{{ 'csv_column_roles_text'|_ }}

+
+
+ +
+
+
+ + +
+
+
+
+

{{ 'csv_column_roles_table'|_ }}

+
+
+ + + + + + + + + + + {% for i in 0..data.columnCount %} + + + + + + + {% endfor %} + + +
{{ 'csv_column_name'|_ }}{{ 'csv_column_example'|_ }}{{ 'csv_column_role'|_ }}{{ 'csv_do_map_value'|_ }}
Column #{{ loop.index }} + {% if data.columns[i]|length == 0 %} + No example data available + {% else %} + {% for example in data.columns[i] %} + {{ example }}
+ {% endfor %} + {% endif %} + + 		+ {{ Form.select(('role['~index~']'), data.available_roles,data.set_roles[index],{class: 'form-control'}) }} + + {# Form.checkbox(('map['~index~']'),1,map[index]) #} +
+ + +
+
+
+
+ +
+
+
+
+ {{ 'csv_go_back'|_ }} + +
+
+
+
+
+ + +{% endblock %}