IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2024-12-28 09:51:21 -06:00
Code Issues Packages Projects Releases Wiki Activity

First (almost) functional CSV importer.

Browse Source
This commit is contained in:
James Cole 2015-07-04 07:53:37 +02:00
parent 26c9b2c353
commit d2c018f7da
12 changed files with 664 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

468
app/Http/Controllers/CsvController.php
View File

@ -9,10 +9,16 @@
namespace FireflyIII\Http\Controllers;
use Auth;
use Carbon\Carbon;
use Config;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Account;
use FireflyIII\Models\TransactionCurrency;
use Illuminate\Http\Request;
use Input;
use League\Csv\Reader;
use Log;
use Redirect;
use Session;
use View;
@ -37,75 +43,465 @@ class CsvController extends Controller
}
/**
* Define column roles and mapping.
*
*
* STEP THREE
*
* @return View
*/
public function columnRoles()
{
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
return Redirect::route('csv.index');
}
}
$subTitle = trans('firefly.csv_process');
$fullPath = Session::get('csv-file');
$hasHeaders = Session::get('csv-has-headers');
$content = file_get_contents($fullPath);
$contentDecrypted = Crypt::decrypt($content);
$reader = Reader::createFromString($contentDecrypted);
Log::debug('Get uploaded content from ' . $fullPath);
Log::debug('Strlen of original content is ' . strlen($contentDecrypted));
Log::debug('MD5 of original content is ' . md5($contentDecrypted));
$firstRow = $reader->fetchOne();
$count = count($firstRow);
$headers = [];
for ($i = 1; $i <= $count; $i++) {
$headers[] = trans('firefly.csv_row') . ' #' . $i;
}
if ($hasHeaders) {
$headers = $firstRow;
}
// example data is always the second row:
$example = $reader->fetchOne();
$roles = [];
foreach (Config::get('csv.roles') as $name => $role) {
$roles[$name] = $role['name'];
}
ksort($roles);
return view('csv.column-roles', compact('roles', 'headers', 'example', 'subTitle'));
}
/**
* This method shows the initial upload form.
*
* STEP ONE
*
* @return View
*/
public function index()
{
$subTitle = trans('firefly.csv_import');
Session::forget('csv-date-format');
Session::forget('csv-has-headers');
Session::forget('csv-file');
// can actually upload?
$uploadPossible = is_writable(storage_path('upload'));
$path = storage_path('upload');
return view('csv.index', compact('subTitle', 'uploadPossible', 'path'));
}
/**
* Parse the file.
*
* STEP FOUR
*
* @return \Illuminate\Http\RedirectResponse
*/
public function initialParse()
{
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
return Redirect::route('csv.index');
}
}
$configRoles = Config::get('csv.roles');
$roles = [];
/*
* Store all rows for each column:
*/
if (is_array(Input::get('role'))) {
$roles = [];
foreach (Input::get('role') as $index => $role) {
if ($role != '_ignore') {
$roles[$index] = $role;
}
}
}
/*
* Go back when no roles defined:
*/
if (count($roles) === 0) {
Session::flash('warning', 'Please select some roles.');
return Redirect::route('csv.column-roles');
}
Session::put('csv-roles', $roles);
/*
* Show user map thing:
*/
if (is_array(Input::get('map'))) {
$maps = [];
foreach (Input::get('map') as $index => $map) {
$name = $roles[$index];
if ($configRoles[$name]['mappable']) {
$maps[$index] = $name;
}
}
// redirect to map routine.
Session::put('csv-map', $maps);
return Redirect::route('csv.map');
}
var_dump($roles);
var_dump($_POST);
exit;
}
/**
*
* Map first if necessary,
*
* STEP FIVE.
*
* @return \Illuminate\Http\RedirectResponse|View
* @throws FireflyException
*/
public function map()
{
/*
* Make sure all fields we need are accounted for.
*/
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
return Redirect::route('csv.index');
}
}
/*
* The $map array contains all columns
* the user wishes to map on to data already in the system.
*/
$map = Session::get('csv-map');
/*
* The "options" array contains all options the user has
* per column, where the key represents the column.
*
* For each key there is an array which in turn represents
* all the options available: grouped by ID.
*/
$options = [];
/*
* Loop each field the user whishes to map.
*/
foreach ($map as $index => $columnRole) {
/*
* Depending on the column role, get the relevant data from the database.
* This needs some work to be optimal.
*/
switch ($columnRole) {
default:
throw new FireflyException('Cannot map field of type "' . $columnRole . '".');
break;
case 'account-iban':
// get content for this column.
$content = Auth::user()->accounts()->where('account_type_id', 3)->get(['accounts.*']);
$list = [];
// make user friendly list:
foreach ($content as $account) {
$list[$account->id] = $account->name;
//if(!is_null($account->iban)) {
//$list[$account->id] .= ' ('.$account->iban.')';
//}
}
$options[$index] = $list;
break;
case 'currency-code':
$currencies = TransactionCurrency::get();
$list = [];
foreach ($currencies as $currency) {
$list[$currency->id] = $currency->name . ' (' . $currency->code . ')';
}
$options[$index] = $list;
break;
case 'opposing-name':
// get content for this column.
$content = Auth::user()->accounts()->whereIn('account_type_id', [4, 5])->get(['accounts.*']);
$list = [];
// make user friendly list:
foreach ($content as $account) {
$list[$account->id] = $account->name . ' (' . $account->accountType->type . ')';
}
$options[$index] = $list;
break;
}
}
/*
* After these values are prepped, read the actual CSV file
*/
$content = file_get_contents(Session::get('csv-file'));
$hasHeaders = Session::get('csv-has-headers');
$reader = Reader::createFromString(Crypt::decrypt($content));
$values = [];
/*
* Loop over the CSV and collect mappable data:
*/
foreach ($reader as $index => $row) {
if (($hasHeaders && $index > 1) || !$hasHeaders) {
// collect all map values
foreach ($map as $column => $irrelevant) {
// check if $irrelevant is mappable!
$values[$column][] = $row[$column];
}
}
}
foreach ($values as $column => $found) {
$values[$column] = array_unique($found);
}
return view('csv.map', compact('map', 'options', 'values'));
}
/**
* Finally actually process the CSV file.
*
* STEP SEVEN
*/
public function process()
{
/*
* Make sure all fields we need are accounted for.
*/
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles', 'csv-mapped'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
return Redirect::route('csv.index');
}
}
// loop the original file again:
$content = file_get_contents(Session::get('csv-file'));
$hasHeaders = Session::get('csv-has-headers');
$reader = Reader::createFromString(Crypt::decrypt($content));
// dump stuff
$dateFormat = Session::get('csv-date-format');
$roles = Session::get('csv-roles');
$mapped = Session::get('csv-mapped');
var_dump($roles);
var_dump(Session::get('csv-mapped'));
/*
* Loop over the CSV and collect mappable data:
*/
foreach ($reader as $index => $row) {
if (($hasHeaders && $index > 1) || !$hasHeaders) {
// this is the data we need to store the new transaction:
$amount = 0;
$amountModifier = 1;
$description = '';
$assetAccount = null;
$opposingAccount = null;
$currency = null;
$date = null;
foreach ($row as $index => $value) {
if (isset($roles[$index])) {
switch ($roles[$index]) {
default:
throw new FireflyException('Cannot process role "' . $roles[$index] . '"');
break;
case 'account-iban':
// find ID in "mapped" (if present).
if (isset($mapped[$index])) {
$searchID = $mapped[$index][$value];
$assetAccount = Account::find($searchID);
} else {
// create account
}
break;
case 'opposing-name':
// don't know yet if its going to be a
// revenue or expense account.
$opposingAccount = $value;
break;
case 'currency-code':
// find ID in "mapped" (if present).
if (isset($mapped[$index])) {
$searchValue = $mapped[$index][$value];
$currency = TransactionCurrency::whereCode($searchValue);
} else {
// create account
}
break;
case 'date-transaction':
// unmappable:
$date = Carbon::createFromFormat($dateFormat, $value);
break;
case 'rabo-debet-credet':
if ($value == 'D') {
$amountModifier = -1;
}
break;
case 'amount':
$amount = $value;
break;
case 'description':
$description .= ' ' . $value;
break;
case 'sepa-ct-id':
$description .= ' ' . $value;
break;
}
}
}
// do something with all this data:
// do something.
var_dump($row);
}
}
}
/**
* Store the mapping the user has made. This is
*
* STEP SIX
*/
public function saveMapping()
{
/*
* Make sure all fields we need are accounted for.
*/
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
return Redirect::route('csv.index');
}
}
// save mapping to session.
$mapped = [];
if (!is_array(Input::get('mapping'))) {
Session::flash('warning', 'Invalid mapping.');
return Redirect::route('csv.map');
}
foreach (Input::get('mapping') as $index => $data) {
$mapped[$index] = [];
foreach ($data as $value => $mapping) {
$mapped[$index][$value] = $mapping;
}
}
Session::put('csv-mapped', $mapped);
// proceed to process.
return Redirect::route('csv.process');
}
/**
*
* This method processes the file, puts it away somewhere safe
* and sends you onwards.
*
* STEP TWO
*
* @param Request $request
*
* @return \Illuminate\Http\RedirectResponse
*/
public function upload(Request $request)
{
// possible column roles:
$roles = [
'(ignore this column)',
'Asset account name',
'Expense or revenue account name',
'Amount',
'Date',
'Currency',
'Description',
'Category',
'Budget',
];
if (!$request->hasFile('csv')) {
Session::flash('warning', 'No file uploaded.');
return Redirect::route('csv.index');
}
$dateFormat = Input::get('date_format');
$hasHeaders = intval(Input::get('has_headers')) === 1;
$reader = Reader::createFromPath($request->file('csv')->getRealPath());
$data = $reader->query();
$data->next(); // go to first row:
$count = count($data->current());
$headers = [];
for ($i = 1; $i <= $count; $i++) {
$headers[] = trans('firefly.csv_row') . ' #' . $i;
}
if ($hasHeaders) {
$headers = $data->current();
}
// example data is always the second row:
$data->next();
$example = $data->current();
// store file somewhere temporary (encrypted)?
$time = str_replace(' ', '-', microtime());
$fileName = 'csv-upload-' . Auth::user()->id . '-' . $time . '.csv.encrypted';
$fullPath = storage_path('upload') . DIRECTORY_SEPARATOR . $fileName;
$content = file_get_contents($request->file('csv')->getRealPath());
$content = Crypt::encrypt($content);
Log::debug('Stored uploaded content in ' . $fullPath);
Log::debug('Strlen of uploaded content is ' . strlen($content));
Log::debug('MD5 of uploaded content is ' . md5($content));
$content = Crypt::encrypt($content);
file_put_contents($fullPath, $content);
Session::put('latestCSVUpload', $fullPath);
$subTitle = trans('firefly.csv_process');
return view('csv.upload', compact('headers', 'example', 'roles', 'subTitle'));
Session::put('csv-date-format', $dateFormat);
Session::put('csv-has-headers', $hasHeaders);
Session::put('csv-file', $fullPath);
return Redirect::route('csv.column-roles');
//
//
//
//
// return view('csv.upload', compact('headers', 'example', 'roles', 'subTitle'));
}
}

5
app/Http/routes.php
View File

@ -225,6 +225,11 @@ Route::group(
*/
Route::get('/csv', ['uses' => 'CsvController@index', 'as' => 'csv.index']);
Route::post('/csv/upload', ['uses' => 'CsvController@upload', 'as' => 'csv.upload']);
Route::get('/csv/column_roles', ['uses' => 'CsvController@columnRoles', 'as' => 'csv.column-roles']);
Route::post('/csv/initial_parse', ['uses' => 'CsvController@initialParse', 'as' => 'csv.initial_parse']);
Route::get('/csv/map', ['uses' => 'CsvController@map', 'as' => 'csv.map']);
Route::post('/csv/save_mapping', ['uses' => 'CsvController@saveMapping', 'as' => 'csv.save_mapping']);
Route::get('/csv/process', ['uses' => 'CsvController@process', 'as' => 'csv.process']);
/**
* Currency Controller

113
config/csv.php Normal file
View File

@ -0,0 +1,113 @@
<?php
return [
'roles' => [
'_ignore' => [
'name' => '(ignore this column)',
'mappable' => false,
],
'bill-id' => [
'name' => 'Bill ID (matching Firefly)',
'mappable' => true,
],
'bill-name' => [
'name' => 'Bill name',
'mappable' => true,
],
'currency-id' => [
'name' => 'Currency ID (matching Firefly)',
'mappable' => true,
],
'currency-name' => [
'name' => 'Currency name (matching Firefly)',
'mappable' => true,
],
'currency-code' => [
'name' => 'Currency code (ISO 4217)',
'mappable' => true,
],
'currency-symbol' => [
'name' => 'Currency symbol (matching Firefly)',
'mappable' => true,
],
'description' => [
'name' => 'Description',
'mappable' => false,
],
'date-transaction' => [
'name' => 'Date',
'mappable' => false,
],
'date-rent' => [
'name' => 'Rent calculation date',
'mappable' => false,
],
'budget-id' => [
'name' => 'Budget ID (matching Firefly)',
'mappable' => true,
],
'budget-name' => [
'name' => 'Budget name',
'mappable' => true,
],
'rabo-debet-credet' => [
'name' => 'Rabobank specific debet/credet indicator',
'mappable' => false,
],
'category-id' => [
'name' => 'Category ID (matching Firefly)',
'mappable' => true,
],
'category-name' => [
'name' => 'Category name',
'mappable' => true,
],
'tags-comma' => [
'name' => 'Tags (comma separated)',
'mappable' => true,
],
'tags-space' => [
'name' => 'Tags (space separated)',
'mappable' => true,
],
'account-id' => [
'name' => 'Asset account ID (matching Firefly)',
'mappable' => true,
],
'account-name' => [
'name' => 'Asset account name',
'mappable' => true,
],
'account-iban' => [
'name' => 'Asset account IBAN',
'mappable' => true,
],
'opposing-id' => [
'name' => 'Expense or revenue account ID (matching Firefly)',
'mappable' => true,
],
'opposing-name' => [
'name' => 'Expense or revenue account name',
'mappable' => true,
],
'opposing-iban' => [
'name' => 'Expense or revenue account IBAN',
'mappable' => true,
],
'amount' => [
'name' => 'Amount',
'mappable' => false,
],
'sepa-ct-id' => [
'name' => 'SEPA Credit Transfer end-to-end ID',
'mappable' => false,
],
'sepa-ct-op' => [
'name' => 'SEPA Credit Transfer opposing account',
'mappable' => false,
],
'sepa-db' => [
'name' => 'SEPA Direct Debet',
'mappable' => false,
],
]
];

8
database/seeds/TestDataSeeder.php
View File

@ -98,7 +98,7 @@ class TestDataSeeder extends Seeder
protected function createAssetAccounts()
{
$assets = ['MyBank Checking Account', 'Savings', 'Shared', 'Creditcard'];
$assets = ['MyBank Checking Account', 'Savings', 'Shared', 'Creditcard', 'Emergencies', 'STE'];
$assetMeta = [
[
'accountRole' => 'defaultAsset',
@ -114,6 +114,12 @@ class TestDataSeeder extends Seeder
'ccMonthlyPaymentDate' => '2015-05-27',
'ccType' => 'monthlyFull'
],
[
'accountRole' => 'savingAsset',
],
[
'accountRole' => 'savingAsset',
],
];

5
resources/lang/en/firefly.php
View File

@ -26,8 +26,11 @@ return [
'csv_upload_form' => 'Upload form',
'upload_csv_file' => 'Upload CSV file',
'csv_header_help' => 'Check this when bla bla',
'csv_date_help' => 'Date time format in your CSV. Follow the format like <a href="https://secure.' .
'php.net/manual/en/datetime.createfromformat.php#refsect1-datetime.createfromformat-parameters">this' .
' page</a> indicates.',
'csv_row' => 'row',
'upload_not_writeable' => 'Cannot write to the path mentioned here. Cannot upload',
'upload_not_writeable' => 'Cannot write to the path mentioned here. Cannot upload',
// create new stuff:
'create_new_withdrawal' => 'Create new withdrawal',

1
resources/lang/en/form.php
View File

@ -47,6 +47,7 @@ return [
'code' => 'Code',
'csv' => 'CSV file',
'has_headers' => 'Headers',
'date_format' => 'Date format',
'store_new_withdrawal' => 'Store new withdrawal',
'store_new_deposit' => 'Store new deposit',

1
resources/lang/nl/form.php
View File

@ -47,6 +47,7 @@ return [
'code' => 'Code',
'csv' => 'CSV-bestand',
'has_headers' => 'Eerste rij zijn kolomnamen',
'date_format' => 'Datumformaat',
'store_new_withdrawal' => 'Nieuwe uitgave opslaan',
'store_new_deposit' => 'Nieuwe inkomsten opslaan',

18
resources/twig/csv/upload.twig → resources/twig/csv/column-roles.twig
View File

@ -20,7 +20,9 @@
</div>
<div class="box-body">
{{ 'csv_process_text'|_ }}
<p>{{ 'csv_process_text'|_ }}</p>
<h4>{{ 'csv_more_information' }}</h4>
<p>{{ 'csv_more_information_text'|_ }}</p>
</div>
</div>
@ -39,7 +41,8 @@
</div>
<div class="box-body">
<form action="{{ route('csv.initial_parse') }}" method="post">
<input type="hidden" name="_token" value="{{ csrf_token() }}" />
<table class="table">
<thead>
<th>{{ 'cvs_column_name'|_ }}</th>
@ -52,16 +55,21 @@
<td>{{ header }}</td>
<td>{{ example[index] }}</td>
<td>
{{ Form.select(('role_'~index), roles) }}
{{ Form.select(('role['~index~']'), roles) }}
</td>
<td>
{{ Form.checkbox(('map_'~index),false) }}
{{ Form.checkbox(('map['~index~']'),1,false) }}
</td>
</tr>
{% endfor %}
</table>
<p>
<button type="submit" class="btn btn-success">
{{ 'process_csv_file'|_ }}
</button>
</p>
</form>
</div>
</div>

3
resources/twig/csv/index.twig
View File

@ -21,6 +21,8 @@
</div>
<div class="box-body">
{{ 'csv_index_text'|_ }}
<h4>{{ 'csv_more_information' }}</h4>
{{ 'csv_more_information_text'|_ }}
</div>
</div>
@ -44,6 +46,7 @@
<input type="hidden" name="_token" value="{{ csrf_token() }}" />
{{ ExpandedForm.checkbox('has_headers',false,null,{helpText: 'csv_header_help'|_}) }}
{{ ExpandedForm.text('date_format','Ymd',{helpText: 'csv_date_help'|_}) }}
{{ ExpandedForm.file('csv') }}

83
resources/twig/csv/map.twig Normal file
View File

@ -0,0 +1,83 @@
{% extends "./layout/default.twig" %}
{% block breadcrumbs %}
{{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }}
{% endblock %}
{% block content %}
<div class="row">
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ 'csv_map'|_ }}</h3>
<!-- ACTIONS MENU -->
<div class="box-tools pull-right">
<button class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i></button>
</div>
</div>
<div class="box-body">
<p>{{ 'csv_map_text'|_ }}</p>
<h4>{{ 'csv_more_information' }}</h4>
<p>{{ 'csv_more_information_text'|_ }}</p>
</div>
</div>
</div>
</div>
<form action="{{ route('csv.save_mapping') }}" method="post">
<input type="hidden" name="_token" value="{{ csrf_token() }}"/>
{% for index,columnName in map %}
<div class="row">
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ columnName }}</h3>
<!-- ACTIONS MENU -->
<div class="box-tools pull-right">
<button class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i></button>
</div>
</div>
<div class="box-body">
<table class="table">
<thead>
<th>{{ 'cvs_field_value'|_ }}</th>
<th>{{ 'csv_field_mapped_to'|_ }}</th>
</thead>
<tbody>
{% for value in values[index] %}
<tr>
<td>{{ value }}</td>
<td>
{{ Form.select('mapping['~index~']['~value~']',options[index]) }}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
{% endfor %}
<p>
<button type="submit" class="btn btn-success">
{{ 'save_mapping'|_ }}
</button>
</p>
</form>
{% endblock %}

2
resources/twig/form/help.twig
View File

@ -1,3 +1,3 @@
{% if options.helpText %}
<p class="help-block">{{ options.helpText }}</p>
<p class="help-block">{{ options.helpText|raw }}</p>
{% endif %}

1
resources/twig/form/text.twig
View File

@ -3,6 +3,7 @@
<div class="col-sm-8">
{{ Form.input('text', name, value, options) }}
{% include 'form/help.twig' %}
{% include 'form/feedback.twig' %}
</div>
</div>