IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

Greatly expanded the CSV routine.

Browse Source
This commit is contained in:
James Cole 2015-07-05 06:18:02 +02:00
parent d2c018f7da
commit 65122f0144
22 changed files with 1282 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
app/Helpers/Csv/Converter/AccountIban.php Normal file
View File

@ -0,0 +1,46 @@
<?php
/**
* Created by PhpStorm.
* User: sander
* Date: 05/07/15
* Time: 05:49
*/
namespace FireflyIII\Helpers\Csv\Converter;
use Auth;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
/**
* Class AccountIban
*
* @package FireflyIII\Helpers\Csv\Converter
*/
class AccountIban extends BasicConverter implements ConverterInterface
{
/**
* @return Account|null
*/
public function convert()
{
// is mapped? Then it's easy!
if (isset($this->mapped[$this->index][$this->value])) {
$account = Auth::user()->accounts()->find($this->mapped[$this->index][$this->value]);
} else {
// find or create new account:
$accountType = AccountType::where('type', 'Asset account')->first();
$account = Account::firstOrCreateEncrypted(
[
'name' => $this->value,
//'iban' => $this->value,
'user_id' => Auth::user()->id,
'account_type_id' => $accountType->id
]
);
}
return $account;
}
}

32
app/Helpers/Csv/Converter/Amount.php Normal file
View File

@ -0,0 +1,32 @@
<?php
/**
* Created by PhpStorm.
* User: sander
* Date: 05/07/15
* Time: 05:49
*/
namespace FireflyIII\Helpers\Csv\Converter;
use FireflyIII\Models\Account;
/**
* Class Amount
*
* @package FireflyIII\Helpers\Csv\Converter
*/
class Amount extends BasicConverter implements ConverterInterface
{
/**
* @return Account|null
*/
public function convert()
{
if (is_numeric($this->value)) {
return $this->value;
}
return 0;
}
}

101
app/Helpers/Csv/Converter/BasicConverter.php Normal file
View File

@ -0,0 +1,101 @@
<?php
namespace FireflyIII\Helpers\Csv\Converter;
/**
* Class BasicConverter
*
* @package FireflyIII\Helpers\Csv\Converter
*/
class BasicConverter
{
/** @var array */
protected $data;
protected $index;
/** @var array */
protected $mapped;
protected $role;
protected $value;
/**
* @return array
*/
public function getData()
{
return $this->data;
}
/**
* @param array $data
*/
public function setData($data)
{
$this->data = $data;
}
/**
* @return mixed
*/
public function getIndex()
{
return $this->index;
}
/**
* @param mixed $index
*/
public function setIndex($index)
{
$this->index = $index;
}
/**
* @return array
*/
public function getMapped()
{
return $this->mapped;
}
/**
* @param array $mapped
*/
public function setMapped($mapped)
{
$this->mapped = $mapped;
}
/**
* @return mixed
*/
public function getRole()
{
return $this->role;
}
/**
* @param mixed $role
*/
public function setRole($role)
{
$this->role = $role;
}
/**
* @return mixed
*/
public function getValue()
{
return $this->value;
}
/**
* @param mixed $value
*/
public function setValue($value)
{
$this->value = $value;
}
}

51
app/Helpers/Csv/Converter/ConverterInterface.php Normal file
View File

@ -0,0 +1,51 @@
<?php
/**
* Created by PhpStorm.
* User: sander
* Date: 05/07/15
* Time: 05:42
*/
namespace FireflyIII\Helpers\Csv\Converter;
/**
* Interface ConverterInterface
*
* @package FireflyIII\Helpers\Csv\Converter
*/
interface ConverterInterface
{
/**
* @return mixed
*/
public function convert();
/**
* @param $index
*/
public function setIndex($index);
/**
* @param $mapped
*/
public function setMapped($mapped);
/**
* @param $role
*/
public function setRole($role);
/**
* @param $value
*/
public function setValue($value);
/**
* @param array $data
*
* @return mixed
*/
public function setData(array $data);
}

27
app/Helpers/Csv/Converter/CurrencyCode.php Normal file
View File

@ -0,0 +1,27 @@
<?php
namespace FireflyIII\Helpers\Csv\Converter;
use FireflyIII\Models\TransactionCurrency;
/**
* Class CurrencyCode
*
* @package FireflyIII\Helpers\Csv\Converter
*/
class CurrencyCode extends BasicConverter implements ConverterInterface
{
/**
* @return mixed|static
*/
public function convert()
{
if (isset($this->mapped[$this->index][$this->value])) {
$currency = TransactionCurrency::find($this->mapped[$this->index][$this->value]);
} else {
$currency = TransactionCurrency::whereCode($this->value)->first();
}
return $currency;
}
}

33
app/Helpers/Csv/Converter/Date.php Normal file
View File

@ -0,0 +1,33 @@
<?php
/**
* Created by PhpStorm.
* User: sander
* Date: 05/07/15
* Time: 05:49
*/
namespace FireflyIII\Helpers\Csv\Converter;
use Carbon\Carbon;
use Session;
/**
* Class Date
*
* @package FireflyIII\Helpers\Csv\Converter
*/
class Date extends BasicConverter implements ConverterInterface
{
/**
* @return Carbon
*/
public function convert()
{
$format = Session::get('csv-date-format');
$date = Carbon::createFromFormat($format, $this->value);
return $date;
}
}

22
app/Helpers/Csv/Converter/Ignore.php Normal file
View File

@ -0,0 +1,22 @@
<?php
namespace FireflyIII\Helpers\Csv\Converter;
use FireflyIII\Models\Account;
/**
* Class Amount
*
* @package FireflyIII\Helpers\Csv\Converter
*/
class Ignore extends BasicConverter implements ConverterInterface
{
/**
* @return Account|null
*/
public function convert()
{
return null;
}
}

27
app/Helpers/Csv/Converter/RabobankDebetCredit.php Normal file
View File

@ -0,0 +1,27 @@
<?php
/**
* Created by PhpStorm.
* User: sander
* Date: 05/07/15
* Time: 06:12
*/
namespace FireflyIII\Helpers\Csv\Converter;
class RabobankDebetCredit extends BasicConverter implements ConverterInterface
{
/**
* @return mixed
*/
public function convert()
{
if ($this->value == 'D') {
return -1;
}
return 1;
}
}

235
app/Helpers/Csv/Data.php Normal file
View File

@ -0,0 +1,235 @@
<?php
namespace FireflyIII\Helpers\Csv;
use Crypt;
use League\Csv\Reader;
use Session;
/**
* Class Data
*
* @package FireflyIII\Helpers\Csv
*/
class Data
{
/** @var string */
protected $csvFileContent;
/** @var string */
protected $csvFileLocation;
/** @var string */
protected $dateFormat;
/** @var bool */
protected $hasHeaders;
/** @var array */
protected $map;
/** @var array */
protected $mapped;
/** @var Reader */
protected $reader;
/** @var array */
protected $roles;
/**
*
*/
public function __construct()
{
$this->sessionHasHeaders();
$this->sessionDateFormat();
$this->sessionCsvFileLocation();
$this->sessionMap();
$this->sessionRoles();
$this->sessionMapped();
}
protected function sessionHasHeaders()
{
if (Session::has('csv-has-headers')) {
$this->hasHeaders = (bool)Session::get('csv-has-headers');
}
}
protected function sessionDateFormat()
{
if (Session::has('csv-date-format')) {
$this->dateFormat = (string)Session::get('csv-date-format');
}
}
protected function sessionCsvFileLocation()
{
if (Session::has('csv-file')) {
$this->csvFileLocation = (string)Session::get('csv-file');
}
}
protected function sessionMap()
{
if (Session::has('csv-map')) {
$this->map = (array)Session::get('csv-map');
}
}
protected function sessionRoles()
{
if (Session::has('csv-roles')) {
$this->roles = (array)Session::get('csv-roles');
}
}
protected function sessionMapped()
{
if (Session::has('csv-mapped')) {
$this->mapped = (array)Session::get('csv-mapped');
}
}
/**
* @return string
*/
public function getDateFormat()
{
return $this->dateFormat;
}
/**
* @param mixed $dateFormat
*/
public function setDateFormat($dateFormat)
{
Session::put('csv-date-format', $dateFormat);
$this->dateFormat = $dateFormat;
}
/**
* @return bool
*/
public function getHasHeaders()
{
return $this->hasHeaders;
}
/**
* @param bool $hasHeaders
*/
public function setHasHeaders($hasHeaders)
{
Session::put('csv-has-headers', $hasHeaders);
$this->hasHeaders = $hasHeaders;
}
/**
* @return array
*/
public function getMap()
{
return $this->map;
}
/**
* @param array $map
*/
public function setMap(array $map)
{
Session::put('csv-map', $map);
$this->map = $map;
}
/**
* @return array
*/
public function getMapped()
{
return $this->mapped;
}
/**
* @param array $mapped
*/
public function setMapped(array $mapped)
{
Session::put('csv-mapped', $mapped);
$this->mapped = $mapped;
}
/**
* @return Reader
*/
public function getReader()
{
if (strlen($this->csvFileContent) === 0) {
$this->loadCsvFile();
}
if (is_null($this->reader)) {
$this->reader = Reader::createFromString($this->getCsvFileContent());
}
return $this->reader;
}
protected function loadCsvFile()
{
$file = $this->getCsvFileLocation();
$content = file_get_contents($file);
$contentDecrypted = Crypt::decrypt($content);
$this->setCsvFileContent($contentDecrypted);
}
/**
* @return string
*/
public function getCsvFileLocation()
{
return $this->csvFileLocation;
}
/**
* @param string $csvFileLocation
*/
public function setCsvFileLocation($csvFileLocation)
{
Session::put('csv-file', $csvFileLocation);
$this->csvFileLocation = $csvFileLocation;
}
/**
* @return string
*/
public function getCsvFileContent()
{
return $this->csvFileContent;
}
/**
* @param string $csvFileContent
*/
public function setCsvFileContent($csvFileContent)
{
$this->csvFileContent = $csvFileContent;
}
/**
* @return array
*/
public function getRoles()
{
return $this->roles;
}
/**
* @param array $roles
*/
public function setRoles(array $roles)
{
Session::put('csv-roles', $roles);
$this->roles = $roles;
}
}

54
app/Helpers/Csv/DataGrabber.php Normal file
View File

@ -0,0 +1,54 @@
<?php
namespace FireflyIII\Helpers\Csv;
use Auth;
use FireflyIII\Models\Account;
use FireflyIII\Models\TransactionCurrency;
use Illuminate\Database\Eloquent\Relations\HasMany;
/**
* Class DataGrabber
*
* Class dedicated to retreiving all sorts of data related to the CSV import.
*
* @package FireflyIII\Helpers\Csv
*/
class DataGrabber
{
/**
* @return array
*/
public function getAssetAccounts()
{
$result = Auth::user()->accounts()->with(
['accountmeta' => function (HasMany $query) {
$query->where('name', 'accountRole');
}]
)->accountTypeIn(['Default account', 'Asset account'])->orderBy('accounts.name', 'ASC')->get(['accounts.*']);
$list = [];
/** @var Account $account */
foreach ($result as $account) {
$list[$account->id] = $account->name;
}
return $list;
}
/**
* @return array
*/
public function getCurrencies()
{
$currencies = TransactionCurrency::get();
$list = [];
foreach ($currencies as $currency) {
$list[$currency->id] = $currency->name . ' (' . $currency->code . ')';
}
return $list;
}
}

175
app/Helpers/Csv/Importer.php Normal file
View File

@ -0,0 +1,175 @@
<?php
namespace FireflyIII\Helpers\Csv;
use App;
use Carbon\Carbon;
use Config;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Csv\Converter\ConverterInterface;
/**
* Class Importer
*
* @package FireflyIII\Helpers\Csv
*/
class Importer
{
/** @var Data */
protected $data;
/** @var array */
protected $map;
/** @var array */
protected $mapped;
/** @var array */
protected $roles;
/**
* @param $value
*/
public function parseRaboDebetCredit($value)
{
if ($value == 'D') {
return -1;
}
return 1;
}
/**
*
*/
public function run()
{
$this->map = $this->data->getMap();
$this->roles = $this->data->getRoles();
$this->mapped = $this->data->getMapped();
foreach ($this->data->getReader() as $row) {
$this->importRow($row);
}
}
/**
* @param $row
*
* @throws FireflyException
*/
protected function importRow($row)
{
/*
* These fields are necessary to create a new transaction journal. Some are optional:
*/
$data = $this->getFiller();
foreach ($row as $index => $value) {
$role = isset($this->roles[$index]) ? $this->roles[$index] : '_ignore';
$class = Config::get('csv.roles.' . $role . '.converter');
$field = Config::get('csv.roles.' . $role . '.field');
if (is_null($class)) {
throw new FireflyException('No converter for field of type "' . $role . '".');
}
if (is_null($field)) {
throw new FireflyException('No place to store value of type "' . $role . '".');
}
/** @var ConverterInterface $converter */
$converter = App::make('FireflyIII\Helpers\Csv\Converter\\' . $class);
$converter->setData($data); // the complete array so far.
$converter->setIndex($index);
$converter->setValue($value);
$converter->setRole($role);
// if (is_array($field)) {
// $convertResult = $converter->convert();
// foreach ($field as $fieldName) {
// $data[$fieldName] = $convertResult[$fieldName];
// }
// } else {
$data[$field] = $converter->convert();
// }
// case 'description':
// $data['description'] .= ' ' . $value;
// break;
// case '_ignore':
// ignore! (duh)
// break;
// case 'account-iban':
// $data['asset-account'] = $this->findAssetAccount($index, $value);
// break;
// case 'currency-code':
// $data['currency'] = $this->findCurrency($index, $value, $role);
// break;
// case 'date-transaction':
// $data['date'] = $this->parseDate($value);
// break;
// case 'rabo-debet-credit':
// $data['amount-modifier'] = $this->parseRaboDebetCredit($value);
// break;
// default:
// throw new FireflyException('Cannot process row of type "' . $role . '".');
// break;
}
$data = $this->postProcess($data);
var_dump($data);
exit;
}
/**
* @return array
*/
protected function getFiller()
{
return [
'description' => '',
'asset-account' => null,
'date' => null,
'currency' => null,
'amount' => null,
'amount-modifier' => 1,
'ignored' => null,
];
}
/**
* @param array $data
*
* @return array
*/
protected function postProcess(array $data)
{
$data['description'] = trim($data['description']);
return $data;
}
/**
* @param Data $data
*/
public function setData($data)
{
$this->data = $data;
}
/**
* @param $value
*
* @return Carbon
*/
protected function parseDate($value)
{
return Carbon::createFromFormat($this->data->getDateFormat(), $value);
}
}

178
app/Helpers/Csv/Wizard.php Normal file
View File

@ -0,0 +1,178 @@
<?php
namespace FireflyIII\Helpers\Csv;
use Auth;
use Config;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use League\Csv\Reader;
use Session;
/**
* Class Wizard
*
* @package FireflyIII\Helpers\Csv
*/
class Wizard implements WizardInterface
{
/**
* @param Reader $reader
* @param array $map
* @param bool $hasHeaders
*
* @return array
*/
public function getMappableValues($reader, array $map, $hasHeaders)
{
$values = [];
/*
* Loop over the CSV and collect mappable data:
*/
foreach ($reader as $index => $row) {
if (($hasHeaders && $index > 1) || !$hasHeaders) {
// collect all map values
foreach ($map as $column => $irrelevant) {
// check if $irrelevant is mappable!
$values[$column][] = $row[$column];
}
}
}
/*
* Make each one unique.
*/
foreach ($values as $column => $found) {
$values[$column] = array_unique($found);
}
return $values;
}
/**
* @param array $roles
* @param mixed $map
*
* @return array
*/
public function processSelectedMapping(array $roles, $map)
{
$configRoles = Config::get('csv.roles');
$maps = [];
if (is_array($map)) {
foreach ($map as $index => $field) {
if (isset($roles[$index])) {
$name = $roles[$index];
if ($configRoles[$name]['mappable']) {
$maps[$index] = $name;
}
}
}
}
return $maps;
}
/**
* @param mixed $input
*
* @return array
*/
public function processSelectedRoles($input)
{
$roles = [];
/*
* Store all rows for each column:
*/
if (is_array($input)) {
foreach ($input as $index => $role) {
if ($role != '_ignore') {
$roles[$index] = $role;
}
}
}
return $roles;
}
/**
* @param array $fields
*
* @return bool
*/
public function sessionHasValues(array $fields)
{
foreach ($fields as $field) {
if (!Session::has($field)) {
return false;
}
}
return true;
}
/**
* @param array $map
*
* @return array
* @throws FireflyException
*/
public function showOptions(array $map)
{
$dataGrabber = new DataGrabber;
$options = [];
foreach ($map as $index => $columnRole) {
/*
* Depending on the column role, get the relevant data from the database.
* This needs some work to be optimal.
*/
switch ($columnRole) {
default:
throw new FireflyException('Cannot map field of type "' . $columnRole . '".');
break;
case 'account-iban':
$set = $dataGrabber->getAssetAccounts();
break;
case 'currency-code':
$set = $dataGrabber->getCurrencies();
break;
}
/*
* Make select list kind of thing:
*/
$options[$index] = $set;
}
return $options;
}
/**
* @param $path
*
* @return string
*/
public function storeCsvFile($path)
{
$time = str_replace(' ', '-', microtime());
$fileName = 'csv-upload-' . Auth::user()->id . '-' . $time . '.csv.encrypted';
$fullPath = storage_path('upload') . DIRECTORY_SEPARATOR . $fileName;
$content = file_get_contents($path);
$contentEncrypted = Crypt::encrypt($content);
file_put_contents($fullPath, $contentEncrypted);
return $fullPath;
}
}

58
app/Helpers/Csv/WizardInterface.php Normal file
View File

@ -0,0 +1,58 @@
<?php
namespace FireflyIII\Helpers\Csv;
use League\Csv\Reader;
/**
* Interface WizardInterface
*
* @package FireflyIII\Helpers\Csv
*/
interface WizardInterface
{
/**
* @param Reader $reader
* @param array $map
* @param bool $hasHeaders
*
* @return array
*/
public function getMappableValues($reader, array $map, $hasHeaders);
/**
* @param array $roles
* @param mixed $map
*
* @return array
*/
public function processSelectedMapping(array $roles, $map);
/**
* @param mixed $input
*
* @return array
*/
public function processSelectedRoles($input);
/**
* @param array $fields
*
* @return bool
*/
public function sessionHasValues(array $fields);
/**
* @param array $map
*
* @return array
*/
public function showOptions(array $map);
/**
* @param $path
*
* @return string
*/
public function storeCsvFile($path);
}

352
app/Http/Controllers/CsvController.php
View File

@ -8,17 +8,19 @@
namespace FireflyIII\Http\Controllers;
use Auth;
use App;
use Carbon\Carbon;
use Config;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Csv\Data;
use FireflyIII\Helpers\Csv\Importer;
use FireflyIII\Helpers\Csv\WizardInterface;
use FireflyIII\Models\Account;
use FireflyIII\Models\TransactionCurrency;
use Illuminate\Http\Request;
use Input;
use League\Csv\Reader;
use Log;
use Redirect;
use Session;
use View;
@ -31,6 +33,11 @@ use View;
class CsvController extends Controller
{
/** @var Data */
protected $data;
/** @var WizardInterface */
protected $wizard;
/**
*
*/
@ -40,6 +47,9 @@ class CsvController extends Controller
View::share('title', trans('firefly.csv'));
View::share('mainTitleIcon', 'fa-file-text-o');
$this->wizard = App::make('FireflyIII\Helpers\Csv\WizardInterface');
$this->data = App::make('FireflyIII\Helpers\Csv\Data');
}
/**
@ -52,48 +62,81 @@ class CsvController extends Controller
*/
public function columnRoles()
{
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
return Redirect::route('csv.index');
}
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers'];
if (!$this->wizard->sessionHasValues($fields)) {
Session::flash('warning', 'Could not recover upload.');
return Redirect::route('csv.index');
}
$subTitle = trans('firefly.csv_process');
$fullPath = Session::get('csv-file');
$hasHeaders = Session::get('csv-has-headers');
$content = file_get_contents($fullPath);
$contentDecrypted = Crypt::decrypt($content);
$reader = Reader::createFromString($contentDecrypted);
$subTitle = trans('firefly.csv_process');
$firstRow = $this->data->getReader()->fetchOne();
$count = count($firstRow);
$headers = [];
$example = $this->data->getReader()->fetchOne();
$availableRoles = [];
$roles = $this->data->getRoles();
$map = $this->data->getMap();
Log::debug('Get uploaded content from ' . $fullPath);
Log::debug('Strlen of original content is ' . strlen($contentDecrypted));
Log::debug('MD5 of original content is ' . md5($contentDecrypted));
$firstRow = $reader->fetchOne();
$count = count($firstRow);
$headers = [];
for ($i = 1; $i <= $count; $i++) {
$headers[] = trans('firefly.csv_row') . ' #' . $i;
}
if ($hasHeaders) {
if ($this->data->getHasHeaders()) {
$headers = $firstRow;
}
// example data is always the second row:
$example = $reader->fetchOne();
$roles = [];
foreach (Config::get('csv.roles') as $name => $role) {
$roles[$name] = $role['name'];
$availableRoles[$name] = $role['name'];
}
ksort($roles);
ksort($availableRoles);
return view('csv.column-roles', compact('availableRoles', 'map', 'roles', 'headers', 'example', 'subTitle'));
}
return view('csv.column-roles', compact('roles', 'headers', 'example', 'subTitle'));
/**
* Optional download of mapping.
*
* STEP FOUR THREE-A
*/
public function downloadConfig()
{
$fields = ['csv-date-format', 'csv-has-headers'];
if (!$this->wizard->sessionHasValues($fields)) {
Session::flash('warning', 'Could not recover upload.');
return Redirect::route('csv.index');
}
$data = [
'date-format' => Session::get('date-format'),
'has-headers' => Session::get('csv-has-headers')
];
// $fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles', 'csv-mapped'];
if (Session::has('csv-map')) {
$data['map'] = Session::get('csv-map');
}
if (Session::has('csv-roles')) {
$data['roles'] = Session::get('csv-roles');
}
if (Session::has('csv-mapped')) {
$data['mapped'] = Session::get('csv-mapped');
}
$result = json_encode($data, JSON_PRETTY_PRINT);
$name = 'csv-configuration-' . date('Y-m-d') . '.json';
header('Content-disposition: attachment; filename=' . $name);
header('Content-type: application/json');
echo $result;
exit;
}
/**
* @return View
*/
public function downloadConfigPage()
{
return view('csv.download-config');
}
/**
@ -110,6 +153,9 @@ class CsvController extends Controller
Session::forget('csv-date-format');
Session::forget('csv-has-headers');
Session::forget('csv-file');
Session::forget('csv-map');
Session::forget('csv-roles');
Session::forget('csv-mapped');
// can actually upload?
@ -129,28 +175,20 @@ class CsvController extends Controller
public function initialParse()
{
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
if (!$this->wizard->sessionHasValues($fields)) {
Session::flash('warning', 'Could not recover upload.');
return Redirect::route('csv.index');
}
return Redirect::route('csv.index');
}
$configRoles = Config::get('csv.roles');
$roles = [];
/*
* Store all rows for each column:
*/
if (is_array(Input::get('role'))) {
$roles = [];
foreach (Input::get('role') as $index => $role) {
if ($role != '_ignore') {
$roles[$index] = $role;
}
}
}
// process given roles and mapping:
$roles = $this->wizard->processSelectedRoles(Input::get('role'));
$maps = $this->wizard->processSelectedMapping($roles, Input::get('map'));
Session::put('csv-map', $maps);
Session::put('csv-roles', $roles);
/*
* Go back when no roles defined:
*/
@ -159,28 +197,19 @@ class CsvController extends Controller
return Redirect::route('csv.column-roles');
}
Session::put('csv-roles', $roles);
/*
* Show user map thing:
* Continue with map specification when necessary.
*/
if (is_array(Input::get('map'))) {
$maps = [];
foreach (Input::get('map') as $index => $map) {
$name = $roles[$index];
if ($configRoles[$name]['mappable']) {
$maps[$index] = $name;
}
}
// redirect to map routine.
Session::put('csv-map', $maps);
if (count($maps) > 0) {
return Redirect::route('csv.map');
}
var_dump($roles);
var_dump($_POST);
exit;
/*
* Or simply start processing.
*/
return Redirect::route('csv.process');
}
@ -200,106 +229,42 @@ class CsvController extends Controller
* Make sure all fields we need are accounted for.
*/
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
if (!$this->wizard->sessionHasValues($fields)) {
Session::flash('warning', 'Could not recover upload.');
return Redirect::route('csv.index');
}
return Redirect::route('csv.index');
}
/*
* The $map array contains all columns
* the user wishes to map on to data already in the system.
*/
$map = Session::get('csv-map');
/*
* The "options" array contains all options the user has
* per column, where the key represents the column.
*
* For each key there is an array which in turn represents
* all the options available: grouped by ID.
*
* Aka:
*
* options[column index] = [
* field id => field identifier.
* ]
*/
$options = [];
/*
* Loop each field the user whishes to map.
*/
foreach ($map as $index => $columnRole) {
/*
* Depending on the column role, get the relevant data from the database.
* This needs some work to be optimal.
*/
switch ($columnRole) {
default:
throw new FireflyException('Cannot map field of type "' . $columnRole . '".');
break;
case 'account-iban':
// get content for this column.
$content = Auth::user()->accounts()->where('account_type_id', 3)->get(['accounts.*']);
$list = [];
// make user friendly list:
foreach ($content as $account) {
$list[$account->id] = $account->name;
//if(!is_null($account->iban)) {
//$list[$account->id] .= ' ('.$account->iban.')';
//}
}
$options[$index] = $list;
break;
case 'currency-code':
$currencies = TransactionCurrency::get();
$list = [];
foreach ($currencies as $currency) {
$list[$currency->id] = $currency->name . ' (' . $currency->code . ')';
}
$options[$index] = $list;
break;
case 'opposing-name':
// get content for this column.
$content = Auth::user()->accounts()->whereIn('account_type_id', [4, 5])->get(['accounts.*']);
$list = [];
// make user friendly list:
foreach ($content as $account) {
$list[$account->id] = $account->name . ' (' . $account->accountType->type . ')';
}
$options[$index] = $list;
break;
}
try {
$options = $this->wizard->showOptions($this->data->getMap());
} catch (FireflyException $e) {
return view('error', ['message' => $e->getMessage()]);
}
/*
* After these values are prepped, read the actual CSV file
*/
$content = file_get_contents(Session::get('csv-file'));
$hasHeaders = Session::get('csv-has-headers');
$reader = Reader::createFromString(Crypt::decrypt($content));
$values = [];
$reader = $this->data->getReader();
$map = $this->data->getMap();
$hasHeaders = $this->data->getHasHeaders();
$values = $this->wizard->getMappableValues($reader, $map, $hasHeaders);
$map = $this->data->getMap();
$mapped = $this->data->getMapped();
/*
* Loop over the CSV and collect mappable data:
*/
foreach ($reader as $index => $row) {
if (($hasHeaders && $index > 1) || !$hasHeaders) {
// collect all map values
foreach ($map as $column => $irrelevant) {
// check if $irrelevant is mappable!
$values[$column][] = $row[$column];
}
}
}
foreach ($values as $column => $found) {
$values[$column] = array_unique($found);
}
return view('csv.map', compact('map', 'options', 'values'));
return view('csv.map', compact('map', 'options', 'values', 'mapped'));
}
/**
@ -313,14 +278,24 @@ class CsvController extends Controller
* Make sure all fields we need are accounted for.
*/
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles', 'csv-mapped'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
if (!$this->wizard->sessionHasValues($fields)) {
Session::flash('warning', 'Could not recover upload.');
return Redirect::route('csv.index');
}
return Redirect::route('csv.index');
}
//
$importer = new Importer;
$importer->setData($this->data);
try {
$importer->run();
} catch (FireflyException $e) {
return view('error', ['message' => $e->getMessage()]);
}
exit;
// loop the original file again:
$content = file_get_contents(Session::get('csv-file'));
$hasHeaders = Session::get('csv-has-headers');
@ -331,10 +306,6 @@ class CsvController extends Controller
$roles = Session::get('csv-roles');
$mapped = Session::get('csv-mapped');
var_dump($roles);
var_dump(Session::get('csv-mapped'));
/*
* Loop over the CSV and collect mappable data:
*/
@ -424,13 +395,12 @@ class CsvController extends Controller
* Make sure all fields we need are accounted for.
*/
$fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles'];
foreach ($fields as $field) {
if (!Session::has($field)) {
Session::flash('warning', 'Could not recover upload (' . $field . ' missing).');
if (!$this->wizard->sessionHasValues($fields)) {
Session::flash('warning', 'Could not recover upload.');
return Redirect::route('csv.index');
}
return Redirect::route('csv.index');
}
// save mapping to session.
$mapped = [];
if (!is_array(Input::get('mapping'))) {
@ -448,7 +418,7 @@ class CsvController extends Controller
Session::put('csv-mapped', $mapped);
// proceed to process.
return Redirect::route('csv.process');
return Redirect::route('csv.download-config-page');
}
@ -468,40 +438,46 @@ class CsvController extends Controller
if (!$request->hasFile('csv')) {
Session::flash('warning', 'No file uploaded.');
return Redirect::route('csv.index');
}
/*
* Store CSV and put in session.
*/
$fullPath = $this->wizard->storeCsvFile($request->file('csv')->getRealPath());
$dateFormat = Input::get('date_format');
$hasHeaders = intval(Input::get('has_headers')) === 1;
// store file somewhere temporary (encrypted)?
$time = str_replace(' ', '-', microtime());
$fileName = 'csv-upload-' . Auth::user()->id . '-' . $time . '.csv.encrypted';
$fullPath = storage_path('upload') . DIRECTORY_SEPARATOR . $fileName;
$content = file_get_contents($request->file('csv')->getRealPath());
Log::debug('Stored uploaded content in ' . $fullPath);
Log::debug('Strlen of uploaded content is ' . strlen($content));
Log::debug('MD5 of uploaded content is ' . md5($content));
$content = Crypt::encrypt($content);
file_put_contents($fullPath, $content);
$map = [];
$roles = [];
$mapped = [];
Session::put('csv-date-format', $dateFormat);
Session::put('csv-has-headers', $hasHeaders);
Session::put('csv-file', $fullPath);
/*
* Process config file if present.
*/
if ($request->hasFile('csv_config')) {
$data = file_get_contents($request->file('csv_config')->getRealPath());
$json = json_decode($data, true);
if (!is_null($json)) {
$dateFormat = isset($json['date-format']) ? $json['date-format'] : $dateFormat;
$hasHeaders = isset($json['has-headers']) ? $json['has-headers'] : $hasHeaders;
$map = isset($json['map']) && is_array($json['map']) ? $json['map'] : [];
$mapped = isset($json['mapped']) && is_array($json['mapped']) ? $json['mapped'] : [];
$roles = isset($json['roles']) && is_array($json['roles']) ? $json['roles'] : [];
}
}
$this->data->setCsvFileLocation($fullPath);
$this->data->setDateFormat($dateFormat);
$this->data->setHasHeaders($hasHeaders);
$this->data->setMap($map);
$this->data->setMapped($mapped);
$this->data->setRoles($roles);
return Redirect::route('csv.column-roles');
//
//
//
//
// return view('csv.upload', compact('headers', 'example', 'roles', 'subTitle'));
}
}

3
app/Http/routes.php
View File

@ -228,7 +228,10 @@ Route::group(
Route::get('/csv/column_roles', ['uses' => 'CsvController@columnRoles', 'as' => 'csv.column-roles']);
Route::post('/csv/initial_parse', ['uses' => 'CsvController@initialParse', 'as' => 'csv.initial_parse']);
Route::get('/csv/map', ['uses' => 'CsvController@map', 'as' => 'csv.map']);
Route::get('/csv/download-config', ['uses' => 'CsvController@downloadConfig', 'as' => 'csv.download-config']);
Route::get('/csv/download', ['uses' => 'CsvController@downloadConfigPage', 'as' => 'csv.download-config-page']);
Route::post('/csv/save_mapping', ['uses' => 'CsvController@saveMapping', 'as' => 'csv.save_mapping']);
Route::get('/csv/process', ['uses' => 'CsvController@process', 'as' => 'csv.process']);
/**

3
app/Providers/FireflyServiceProvider.php
View File

@ -91,6 +91,9 @@ class FireflyServiceProvider extends ServiceProvider
$this->app->bind('FireflyIII\Repositories\Tag\TagRepositoryInterface', 'FireflyIII\Repositories\Tag\TagRepository');
$this->app->bind('FireflyIII\Support\Search\SearchInterface', 'FireflyIII\Support\Search\Search');
// CSV import
$this->app->bind('FireflyIII\Helpers\Csv\WizardInterface', 'FireflyIII\Helpers\Csv\Wizard');
// make charts:
// alternative is Google instead of ChartJs
$this->app->bind('FireflyIII\Generator\Chart\Account\AccountChartGenerator', 'FireflyIII\Generator\Chart\Account\ChartJsAccountChartGenerator');

34
config/csv.php
View File

@ -4,6 +4,8 @@ return [
'_ignore' => [
'name' => '(ignore this column)',
'mappable' => false,
'converter' => 'Ignore',
'field' => 'ignored',
],
'bill-id' => [
'name' => 'Bill ID (matching Firefly)',
@ -22,8 +24,10 @@ return [
'mappable' => true,
],
'currency-code' => [
'name' => 'Currency code (ISO 4217)',
'mappable' => true,
'name' => 'Currency code (ISO 4217)',
'mappable' => true,
'converter' => 'CurrencyCode',
'field' => 'currency'
],
'currency-symbol' => [
'name' => 'Currency symbol (matching Firefly)',
@ -34,8 +38,10 @@ return [
'mappable' => false,
],
'date-transaction' => [
'name' => 'Date',
'mappable' => false,
'name' => 'Date',
'mappable' => false,
'converter' => 'Date',
'field' => 'date',
],
'date-rent' => [
'name' => 'Rent calculation date',
@ -49,9 +55,11 @@ return [
'name' => 'Budget name',
'mappable' => true,
],
'rabo-debet-credet' => [
'name' => 'Rabobank specific debet/credet indicator',
'mappable' => false,
'rabo-debet-credit' => [
'name' => 'Rabobank specific debet/credit indicator',
'mappable' => false,
'converter' => 'RabobankDebetCredit',
'field' => 'amount-modifier',
],
'category-id' => [
'name' => 'Category ID (matching Firefly)',
@ -78,8 +86,10 @@ return [
'mappable' => true,
],
'account-iban' => [
'name' => 'Asset account IBAN',
'mappable' => true,
'name' => 'Asset account IBAN',
'mappable' => true,
'converter' => 'AccountIban',
'field' => 'asset-account'
],
'opposing-id' => [
'name' => 'Expense or revenue account ID (matching Firefly)',
@ -94,8 +104,10 @@ return [
'mappable' => true,
],
'amount' => [
'name' => 'Amount',
'mappable' => false,
'name' => 'Amount',
'mappable' => false,
'converter' => 'Amount',
'field' => 'amount',
],
'sepa-ct-id' => [
'name' => 'SEPA Credit Transfer end-to-end ID',

5
resources/twig/csv/column-roles.twig
View File

@ -55,16 +55,17 @@
<td>{{ header }}</td>
<td>{{ example[index] }}</td>
<td>
{{ Form.select(('role['~index~']'), roles) }}
{{ Form.select(('role['~index~']'), availableRoles,roles[index]) }}
</td>
<td>
{{ Form.checkbox(('map['~index~']'),1,false) }}
{{ Form.checkbox(('map['~index~']'),1,map[index]) }}
</td>
</tr>
{% endfor %}
</table>
<p>
<a href="{{ route('csv.index') }}" class="btn btn-danger pull-right">{{ 'go_back'|_ }}</a>
<button type="submit" class="btn btn-success">
{{ 'process_csv_file'|_ }}
</button>

40
resources/twig/csv/download-config.twig Normal file
View File

@ -0,0 +1,40 @@
{% extends "./layout/default.twig" %}
{% block breadcrumbs %}
{{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }}
{% endblock %}
{% block content %}
<div class="row">
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ 'csv_download_config'|_ }}</h3>
<!-- ACTIONS MENU -->
<div class="box-tools pull-right">
<button class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i></button>
</div>
</div>
<div class="box-body">
<p>
{{ 'csv_some_text'|_ }}
</p>
<p>
<a href="{{ route('csv.download-config') }}" class="btn btn-info"><i class="fa fa-download"></i> {{ 'csv_do_download_config'|_ }}</a>
</p>
<p>
{{ 'csv_more_information_text'|_ }}
</p>
<p>
<a href="{{ route('csv.process') }}" class="btn btn-lg btn-success"><i class="fa fa-gears"></i> {{ 'csv_do_process'|_ }}</a>
</p>
</div>
</div>
</div>
</div>
{% endblock %}

5
resources/twig/csv/index.twig
View File

@ -29,7 +29,7 @@
</div>
</div>
<div class="row">
<div class="col-lg-6">
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ 'csv_upload_form'|_ }}</h3>
@ -50,6 +50,8 @@
{{ ExpandedForm.file('csv') }}
{{ ExpandedForm.file('csv_config') }}
{% if uploadPossible %}
<div class="form-group" id="csv_holder">
<div class="col-sm-4">
@ -57,6 +59,7 @@
</div>
<div class="col-sm-8">
<button type="submit" class="btn btn-success">
{{ 'upload_csv_file'|_ }}
</button>

7
resources/twig/csv/map.twig
View File

@ -20,10 +20,7 @@
</div>
<div class="box-body">
<p>{{ 'csv_map_text'|_ }}</p>
<h4>{{ 'csv_more_information' }}</h4>
<p>{{ 'csv_more_information_text'|_ }}</p>
<a href="{{ route('csv.download-config') }}">Download config for use again</a>
</div>
</div>
@ -57,7 +54,7 @@
<tr>
<td>{{ value }}</td>
<td>
{{ Form.select('mapping['~index~']['~value~']',options[index]) }}
{{ Form.select('mapping['~index~']['~value~']',options[index], mapped[index][value]) }}
</td>
</tr>
{% endfor %}

4
resources/twig/error.twig
View File

@ -4,9 +4,7 @@
<div class="row">
<div class="col-lg-12 col-md-12 col-sm-12">
<h1 class="text-danger">Firefly<br/>
<small>Error</small>
</h1>
<h3 class="text-danger">Sorry, an error occurred.</h3>
</div>
</div>