IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-01-23 23:13:18 -06:00
Code Issues Packages Projects Releases Wiki Activity

Lots of new code to test the import routine.

Browse Source 
Signed-off-by: James Cole <thegrumpydictator@gmail.com>
This commit is contained in:
James Cole 2016-08-11 10:21:32 +02:00
parent efe9933721
commit 186b704509
No known key found for this signature in database
GPG Key ID: C16961E655E74B5E
27 changed files with 1315 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
app/Console/Commands/EncryptFile.php Normal file
View File

@ -0,0 +1,69 @@
<?php
/**
* EncryptFile.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms
* of the MIT license. See the LICENSE file for details.
*/
declare(strict_types = 1);
namespace FireflyIII\Console\Commands;
use Crypt;
use Illuminate\Console\Command;
/**
* Class EncryptFile
*
* @package FireflyIII\Console\Commands
*/
class EncryptFile extends Command
{
/**
* The console command description.
*
* @var string
*/
protected $description = 'Encrypts a file and places it in the storage/upload directory.';
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'firefly:encrypt {file} {key}';
/**
* Create a new command instance.
*
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$file = $this->argument('file');
if (!file_exists($file)) {
$this->error(sprintf('File "%s" does not seem to exist.', $file));
return;
}
$content = file_get_contents($file);
$content = Crypt::encrypt($content);
$newName = $this->argument('key') . '.upload';
$path = storage_path('upload') . '/' . $newName;
file_put_contents($path, $content);
$this->line(sprintf('Encrypted "%s" and put it in "%s"', $file, $path));
}
}

2
app/Console/Kernel.php
View File

@ -11,6 +11,7 @@ declare(strict_types = 1);
namespace FireflyIII\Console;
use FireflyIII\Console\Commands\EncryptFile;
use FireflyIII\Console\Commands\Import;
use FireflyIII\Console\Commands\UpgradeFireflyInstructions;
use FireflyIII\Console\Commands\VerifyDatabase;
@ -53,5 +54,6 @@ class Kernel extends ConsoleKernel
UpgradeFireflyInstructions::class,
VerifyDatabase::class,
Import::class,
EncryptFile::class,
];
}

17
app/Crud/Account/AccountCrud.php
View File

@ -103,7 +103,7 @@ class AccountCrud implements AccountCrudInterface
}
/** @var Collection $accounts */
$accounts = $query->get();
$accounts = $query->get(['accounts.*']);
if ($accounts->count() > 0) {
return $accounts->first();
}
@ -126,7 +126,7 @@ class AccountCrud implements AccountCrudInterface
$query->whereIn('account_types.type', $types);
}
$accounts = $query->get();
$accounts = $query->get(['accounts.*']);
/** @var Account $account */
foreach ($accounts as $account) {
if ($account->iban === $iban) {
@ -145,20 +145,23 @@ class AccountCrud implements AccountCrudInterface
*/
public function findByName(string $name, array $types): Account
{
$query = $this->user->accounts()->where('iban', '!=', "");
$query = $this->user->accounts();
Log::debug('Now in findByName()', ['name' => $name, 'types' => $types]);
if (count($types) > 0) {
$query->leftJoin('account_types', 'accounts.account_type_id', '=', 'account_types.id');
$query->whereIn('account_types.type', $types);
}
$accounts = $query->get();
$accounts = $query->get(['accounts.*']);
Log::debug(sprintf('Total set count is %d ', $accounts->count()));
/** @var Account $account */
foreach ($accounts as $account) {
if ($account->name === $name) {
return $account;
Log::debug('Account name is an exact match. ', ['db' => $account->name, 'source' => $name,'id' => $account->id]);
}
}
Log::warning('Found nothing in findByName()', ['name' => $name, 'types' => $types]);
return new Account;
}
@ -285,11 +288,11 @@ class AccountCrud implements AccountCrudInterface
{
$type = AccountType::whereType($type)->first();
if (!is_null($type)) {
$account->account_type_id = $type->id;
$account->accountType()->associate($type);
$account->save();
}
return $account;
return $this->find($account->id);
}

1
app/Http/Controllers/Auth/AuthController.php
View File

@ -64,6 +64,7 @@ class AuthController extends Controller
*/
public function login(Request $request)
{
$this->validate($request, [$this->loginUsername() => 'required', 'password' => 'required',]);
$throttles = $this->isUsingThrottlesLoginsTrait();

1
app/Http/Controllers/HomeController.php
View File

@ -119,6 +119,7 @@ class HomeController extends Controller
*/
public function index(ARI $repository, AccountCrudInterface $crud)
{
$types = config('firefly.accountTypesByIdentifier.asset');
$count = $repository->countAccounts($types);

1
app/Http/Controllers/NewUserController.php
View File

@ -42,6 +42,7 @@ class NewUserController extends Controller
*/
public function index(ARI $repository)
{
View::share('title', 'Welcome to Firefly!');
View::share('mainTitleIcon', 'fa-fire');

6
app/Import/Converter/AssetAccountName.php
View File

@ -36,6 +36,7 @@ class AssetAccountName extends BasicConverter implements ConverterInterface
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
@ -49,6 +50,7 @@ class AssetAccountName extends BasicConverter implements ConverterInterface
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
@ -56,7 +58,7 @@ class AssetAccountName extends BasicConverter implements ConverterInterface
// not mapped? Still try to find it first:
$account = $repository->findByName($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
Log::debug('Found asset account by name', ['value' => $value, 'id' => $account->id]);
return $account;
}
@ -68,6 +70,8 @@ class AssetAccountName extends BasicConverter implements ConverterInterface
);
$this->setCertainty(100);
Log::debug('Created new asset account ', ['name' => $account->name, 'id' => $account->id]);
return $account;

2
app/Import/Converter/BillName.php
View File

@ -66,7 +66,7 @@ class BillName extends BasicConverter implements ConverterInterface
'name' => $value,
'match' => $value,
'amount_min' => 1,
'user_id' => $this->user->id,
'user' => $this->user->id,
'amount_max' => 10,
'date' => date('Ymd'),
'repeat_freq' => 'monthly',

2
app/Import/Converter/BudgetName.php
View File

@ -64,7 +64,7 @@ class BudgetName extends BasicConverter implements ConverterInterface
$budget = $repository->store(
[
'name' => $value,
'user_id' => $this->user->id,
'user' => $this->user->id,
]
);
$this->setCertainty(100);

2
app/Import/Converter/CategoryName.php
View File

@ -64,7 +64,7 @@ class CategoryName extends BasicConverter implements ConverterInterface
$category = $repository->store(
[
'name' => $value,
'user_id' => $this->user->id,
'user' => $this->user->id,
]
);
$this->setCertainty(100);

4
app/Import/Converter/OpposingAccountName.php
View File

@ -54,7 +54,7 @@ class OpposingAccountName extends BasicConverter implements ConverterInterface
// not mapped? Still try to find it first:
$account = $repository->findByName($value, []);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
Log::debug('Found opposing account by name', ['id' => $account->id]);
Log::warning(
'The match between name and account is uncertain because the type of transactions may not have been determined.',
['id' => $account->id, 'name' => $value]
@ -70,6 +70,8 @@ class OpposingAccountName extends BasicConverter implements ConverterInterface
);
$this->setCertainty(100);
Log::debug('Created new opposing account ', ['name' => $account->name, 'id' => $account->id]);
return $account;
}
}

13
app/Import/ImportStorage.php
View File

@ -58,6 +58,7 @@ class ImportStorage
public function store()
{
foreach ($this->entries as $entry) {
Log::debug('--- import store start ---');
$this->storeSingle($entry);
}
@ -80,9 +81,17 @@ class ImportStorage
/**
* @param ImportEntry $entry
*
* @throws FireflyException
*/
private function storeSingle(ImportEntry $entry)
{
if ($entry->valid === false) {
Log::error('Cannot import entry, because valid=false');
return;
}
Log::debug('Going to store entry!');
$billId = is_null($entry->fields['bill']) ? null : $entry->fields['bill']->id;
$journalData = [
@ -145,7 +154,8 @@ class ImportStorage
$one = Transaction::create($sourceData);
$two = Transaction::create($destinationData);
Log::debug('Created transactions', ['source' => $one->id,'destination' => $two->id]);
Log::debug('Created transaction 1', ['id' => $one->id, 'account' => $one->account_id,'account_name' => $source->name]);
Log::debug('Created transaction 2', ['id' => $two->id, 'account' => $two->account_id,'account_name' => $destination->name]);
$journal->completed = 1;
$journal->save();
@ -153,6 +163,5 @@ class ImportStorage
// now attach budget and so on.
}
}

116
app/Import/ImportValidator.php
View File

@ -50,8 +50,10 @@ class ImportValidator
*/
public function clean(): Collection
{
$newCollection = new Collection;
/** @var ImportEntry $entry */
foreach ($this->entries as $entry) {
Log::debug('--- import validator start ---');
/*
* X Adds the date (today) if no date is present.
* X Determins the types of accounts involved (asset, expense, revenue).
@ -59,15 +61,20 @@ class ImportValidator
* - Determins the currency of the transaction.
* X Adds a default description if there isn't one present.
*/
$this->checkAmount($entry);
$this->setDate($entry);
$this->setAssetAccount($entry);
$this->setOpposingAccount($entry);
$this->cleanDescription($entry);
$this->setTransactionType($entry);
$this->setTransactionCurrency($entry);
$entry = $this->checkAmount($entry);
$entry = $this->setDate($entry);
$entry = $this->setAssetAccount($entry);
Log::debug(sprintf('Opposing account is of type %s', $entry->fields['opposing-account']->accountType->type));
$entry = $this->setOpposingAccount($entry);
Log::debug(sprintf('Opposing account is of type %s', $entry->fields['opposing-account']->accountType->type));
$entry = $this->cleanDescription($entry);
$entry = $this->setTransactionType($entry);
$entry = $this->setTransactionCurrency($entry);
$newCollection->push($entry);
}
return $this->entries;
return $newCollection;
}
/**
@ -88,42 +95,52 @@ class ImportValidator
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function checkAmount(ImportEntry $entry)
private function checkAmount(ImportEntry $entry): ImportEntry
{
if ($entry->fields['amount'] == 0) {
$entry->valid = false;
Log::error('Amount of transaction is zero, cannot handle.');
return $entry;
}
Log::debug('Amount is OK.');
return $entry;
}
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function cleanDescription(ImportEntry $entry)
private function cleanDescription(ImportEntry $entry): ImportEntry
{
if (!isset($entry->fields['description'])) {
Log::debug('Set empty transaction description because field was not set.');
$entry->fields['description'] = '(empty transaction description)';
return;
return $entry;
}
if (is_null($entry->fields['description'])) {
Log::debug('Set empty transaction description because field was null.');
$entry->fields['description'] = '(empty transaction description)';
return;
return $entry;
}
if (strlen($entry->fields['description']) == 0) {
Log::debug('Set empty transaction description because field was empty.');
$entry->fields['description'] = '(empty transaction description)';
return;
return $entry;
}
Log::debug('Transaction description is OK.');
$entry->fields['description'] = trim($entry->fields['description']);
return $entry;
}
/**
@ -136,6 +153,8 @@ class ImportValidator
{
$accountType = $account->accountType->type;
if ($accountType === $type) {
Log::debug(sprintf('Account %s already of type %s', $account->name, $type));
return $account;
}
// find it first by new type:
@ -144,7 +163,9 @@ class ImportValidator
$result = $repository->findByName($account->name, [$type]);
if (is_null($result->id)) {
// can convert account:
Log::debug(sprintf('No account named %s of type %s, will convert.', $account->name, $type));
$result = $repository->updateAccountType($account, $type);
}
return $result;
@ -194,29 +215,36 @@ class ImportValidator
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function setAssetAccount(ImportEntry $entry)
private function setAssetAccount(ImportEntry $entry): ImportEntry
{
if (is_null($entry->fields['asset-account'])) {
if (!is_null($this->defaultImportAccount)) {
Log::debug('Set asset account from default asset account');
$entry->fields['asset-account'] = $this->defaultImportAccount;
return;
return $entry;
}
// default import is null? should not happen. Entry cannot be imported.
// set error message and block.
$entry->valid = false;
Log::error('Cannot import entry. Asset account is NULL and import account is also NULL.');
}
Log::debug('Asset account is OK.');
Log::debug('Asset account is OK.', ['id' => $entry->fields['asset-account']->id, 'name' => $entry->fields['asset-account']->name]);
return $entry;
}
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function setDate(ImportEntry $entry)
private function setDate(ImportEntry $entry): ImportEntry
{
if (is_null($entry->fields['date-transaction'])) {
// empty date field? find alternative.
@ -226,24 +254,26 @@ class ImportValidator
Log::debug(sprintf('Copied date-transaction from %s.', $alternative));
$entry->fields['date-transaction'] = clone $entry->fields[$alternative];
return;
return $entry;
}
}
// date is still null at this point
Log::debug('Set date-transaction to today.');
$entry->fields['date-transaction'] = new Carbon;
return;
return $entry;
}
Log::debug('Date-transaction is OK');
return $entry;
}
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function setOpposingAccount(ImportEntry $entry)
private function setOpposingAccount(ImportEntry $entry): ImportEntry
{
// empty opposing account. Create one based on amount.
if (is_null($entry->fields['opposing-account'])) {
@ -253,12 +283,12 @@ class ImportValidator
Log::debug('Created fallback expense account');
$entry->fields['opposing-account'] = $this->fallbackExpenseAccount();
return;
return $entry;
}
Log::debug('Created fallback revenue account');
$entry->fields['opposing-account'] = $this->fallbackRevenueAccount();
return;
return $entry;
}
// opposing is of type "import". Convert to correct type (by amount):
@ -268,14 +298,14 @@ class ImportValidator
$entry->fields['opposing-account'] = $account;
Log::debug('Converted import account to expense account');
return;
return $entry;
}
if ($type == AccountType::IMPORT && $entry->fields['amount'] > 0) {
$account = $this->convertAccount($entry->fields['opposing-account'], AccountType::REVENUE);
$entry->fields['opposing-account'] = $account;
Log::debug('Converted import account to revenue account');
return;
return $entry;
}
// amount < 0, but opposing is revenue
if ($type == AccountType::REVENUE && $entry->fields['amount'] < 0) {
@ -283,7 +313,7 @@ class ImportValidator
$entry->fields['opposing-account'] = $account;
Log::debug('Converted revenue account to expense account');
return;
return $entry;
}
// amount > 0, but opposing is expense
@ -292,45 +322,65 @@ class ImportValidator
$entry->fields['opposing-account'] = $account;
Log::debug('Converted expense account to revenue account');
return;
return $entry;
}
// account type is OK
Log::debug('Opposing account is OK.');
return $entry;
}
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function setTransactionCurrency(ImportEntry $entry)
private function setTransactionCurrency(ImportEntry $entry): ImportEntry
{
if (is_null($entry->fields['currency'])) {
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$entry->fields['currency'] = $repository->findByCode(env('DEFAULT_CURRENCY', 'EUR'));
Log::debug('Set currency to EUR');
return;
return $entry;
}
Log::debug('Currency is OK');
return $entry;
}
/**
* @param ImportEntry $entry
*
* @return ImportEntry
*/
private function setTransactionType(ImportEntry $entry)
private function setTransactionType(ImportEntry $entry): ImportEntry
{
Log::debug(sprintf('Opposing account is of type %s', $entry->fields['opposing-account']->accountType->type));
$type = $entry->fields['opposing-account']->accountType->type;
switch ($type) {
case AccountType::EXPENSE:
$entry->fields['transaction-type'] = TransactionType::whereType(TransactionType::WITHDRAWAL)->first();
break;
Log::debug('Transaction type is now withdrawal.');
return $entry;
case AccountType::REVENUE:
$entry->fields['transaction-type'] = TransactionType::whereType(TransactionType::DEPOSIT)->first();
break;
Log::debug('Transaction type is now deposit.');
return $entry;
case AccountType::ASSET:
$entry->fields['transaction-type'] = TransactionType::whereType(TransactionType::TRANSFER)->first();
break;
Log::debug('Transaction type is now transfer.');
return $entry;
}
Log::error(sprintf('Opposing account is of type %s, cannot handle this.', $type));
$entry->valid = false;
return $entry;
}

1
app/Import/Importer/CsvImporter.php
View File

@ -48,6 +48,7 @@ class CsvImporter implements ImporterInterface
$collection = new Collection;
foreach ($results as $index => $row) {
if ($index >= $start) {
Log::debug('----- import entry build start --');
Log::debug(sprintf('Now going to import row %d.', $index));
$importEntry = $this->importSingleRow($index, $row);
$collection->push($importEntry);

4
app/Models/Account.php
View File

@ -301,8 +301,8 @@ class Account extends Model
*/
public function setNameAttribute($value)
{
$this->attributes['name'] = Crypt::encrypt($value);
$this->attributes['encrypted'] = true;
$this->attributes['name'] = $value;
$this->attributes['encrypted'] = false;
}
/**

2
app/Providers/CrudServiceProvider.php
View File

@ -58,7 +58,7 @@ class CrudServiceProvider extends ServiceProvider
if (!isset($arguments[0]) && !$app->auth->check()) {
throw new FireflyException('There is no user present.');
}
Log::debug('AccountCrud constructor, run with default arguments.', $arguments);
// Log::debug('AccountCrud constructor, run with default arguments.', $arguments);
return app('FireflyIII\Crud\Account\AccountCrud', $arguments);
}

2
app/Repositories/Bill/BillRepository.php
View File

@ -81,7 +81,7 @@ class BillRepository implements BillRepositoryInterface
*/
public function findByName(string $name) : Bill
{
$bills = $this->user->bills()->get();
$bills = $this->user->bills()->get(['bills.*']);
/** @var Bill $bill */
foreach ($bills as $bill) {

2
app/Repositories/Budget/BudgetRepository.php
View File

@ -94,7 +94,7 @@ class BudgetRepository implements BudgetRepositoryInterface
*/
public function findByName(string $name): Budget
{
$budgets = $this->user->budgets()->get();
$budgets = $this->user->budgets()->get(['budgets.*']);
/** @var Budget $budget */
foreach ($budgets as $budget) {
if ($budget->name === $name) {

2
app/Repositories/Category/CategoryRepository.php
View File

@ -110,7 +110,7 @@ class CategoryRepository implements CategoryRepositoryInterface
*/
public function findByName(string $name) : Category
{
$categories = $this->user->categories()->get();
$categories = $this->user->categories()->get(['categories.*']);
foreach ($categories as $category) {
if ($category->name === $name) {
return $category;

2
app/Repositories/ExportJob/ExportJobRepository.php
View File

@ -99,7 +99,7 @@ class ExportJobRepository implements ExportJobRepositoryInterface
*/
public function findByKey(string $key): ExportJob
{
$result = $this->user->exportJobs()->where('key', $key)->first();
$result = $this->user->exportJobs()->where('key', $key)->first(['export_jobs.*']);
if (is_null($result)) {
return new ExportJob;
}

2
app/Repositories/ImportJob/ImportJobRepository.php
View File

@ -71,7 +71,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
*/
public function findByKey(string $key): ImportJob
{
$result = $this->user->importJobs()->where('key', $key)->first();
$result = $this->user->importJobs()->where('key', $key)->first(['import_jobs.*']);
if (is_null($result)) {
return new ImportJob;
}

21
app/Support/Migration/TestData.php
View File

@ -244,6 +244,26 @@ class TestData
DB::table('categories')->insert($insert);
}
/**
*
*/
private function createImportJobs()
{
$insert = [];
foreach ($this->data['import-jobs'] as $job) {
$insert[] = [
'created_at' => $this->time,
'updated_at' => $this->time,
'user_id' => $job['user_id'],
'file_type' => $job['file_type'],
'key' => $job['key'],
'status' => $job['status'],
'configuration' => json_encode($job['configuration']),
];
}
DB::table('import_jobs')->insert($insert);
}
/**
*
*/
@ -808,6 +828,7 @@ class TestData
$this->createMultiWithdrawals();
$this->createMultiDeposits();
$this->createMultiTransfers();
$this->createImportJobs();
}
}

3
storage/database/.gitignore vendored
View File

@ -1,4 +1,3 @@
*
!.gitignore
!seed.local.json
!seed.split.json
!seed.*.json

72
storage/database/seed.import-test.json Normal file
View File

@ -0,0 +1,72 @@
{
"users": [
{
"email": "thegrumpydictator@gmail.com",
"password": "james"
}
],
"roles": [
{
"user_id": 1,
"role": 1
}
],
"accounts": [],
"account-meta": [],
"bills": [],
"budgets": [],
"budget-limits": [],
"monthly-limits": [],
"categories": [],
"piggy-banks": [],
"piggy-events": [],
"rule-groups": [],
"rules": [],
"rule-triggers": [],
"rule-actions": [],
"tags": [],
"monthly-deposits": [],
"monthly-transfers": [],
"monthly-withdrawals": [],
"attachments": [],
"multi-withdrawals": [],
"multi-deposits": [],
"multi-transfers": [],
"import-jobs": [
{
"user_id": 1,
"key": "testImport",
"file_type": "csv",
"status": "settings_complete",
"configuration": {
"has-headers": false,
"date-format": "Ymd",
"delimiter": ",",
"import-account": 0,
"specifics": [],
"column-count": 7,
"column-roles": [
"account-name",
"opposing-name",
"amount",
"date-transaction",
"category-name",
"budget-name",
"description"
],
"column-do-mapping": [
false,
false,
false,
false,
false,
false,
false
],
"column-roles-complete": false,
"column-mapping-config": [],
"column-mapping-complete": false
}
}
]
}

3
storage/database/seed.local.json
View File

@ -1006,5 +1006,6 @@
9
]
}
]
],
"import-jobs": []
}

3
storage/database/seed.split.json
View File

@ -297,5 +297,6 @@
3
]
}
]
],
"import-jobs": []
}

1011
storage/database/seed.testing.json Normal file
View File

File diff suppressed because it is too large Load Diff