More and improved code for the import routine.

This commit is contained in:
James Cole 2017-06-24 05:49:33 +02:00
parent edb355941c
commit 445dbf8779
No known key found for this signature in database
GPG Key ID: C16961E655E74B5E
32 changed files with 662 additions and 1874 deletions

View File

@ -15,30 +15,26 @@ namespace FireflyIII\Http\Controllers;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Requests\ImportUploadRequest;
use FireflyIII\Import\Configurator\ConfiguratorInterface;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\ImportProcedureInterface;
use FireflyIII\Import\Routine\ImportRoutine;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use Illuminate\Http\Request;
use Illuminate\Http\Response as LaravelResponse;
use Illuminate\Support\Collection;
use Log;
use Response;
use Validator;
use View;
/**
* Class ImportController
* Class ImportController.
*
* @package FireflyIII\Http\Controllers
*/
class ImportController extends Controller
{
/** @var ImportJobRepositoryInterface */
public $repository;
/**
*
*/
@ -50,6 +46,7 @@ class ImportController extends Controller
function ($request, $next) {
View::share('mainTitleIcon', 'fa-archive');
View::share('title', trans('firefly.import_data_full'));
$this->repository = app(ImportJobRepositoryInterface::class);
return $next($request);
}
@ -71,6 +68,8 @@ class ImportController extends Controller
// is the job already configured?
if ($configurator->isJobConfigured()) {
$this->repository->updateStatus($job, 'configured');
return redirect(route('import.status', [$job->key]));
}
$view = $configurator->getNextView();
@ -82,11 +81,11 @@ class ImportController extends Controller
}
/**
* Generate a JSON file of the job's config and send it to the user.
* Generate a JSON file of the job's configuration and send it to the user.
*
* @param ImportJob $job
*
* @return mixed
* @return string
*/
public function download(ImportJob $job)
{
@ -140,70 +139,72 @@ class ImportController extends Controller
/**
* This is step 2. It creates an Import Job. Stores the import.
*
* @param ImportUploadRequest $request
* @param ImportJobRepositoryInterface $repository
* @param UserRepositoryInterface $userRepository
* @param ImportUploadRequest $request
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function initialize(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository)
public function initialize(ImportUploadRequest $request)
{
Log::debug('Now in initialize()');
// create import job:
$type = $request->get('import_file_type');
$job = $repository->create($type);
$job = $this->repository->create($type);
Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]);
// process file:
$repository->processFile($job, $request->files->get('import_file'));
$this->repository->processFile($job, $request->files->get('import_file'));
// process config, if present:
if ($request->files->has('configuration_file')) {
$repository->processConfiguration($job, $request->files->get('configuration_file'));
$this->repository->processConfiguration($job, $request->files->get('configuration_file'));
}
$this->repository->updateStatus($job, 'initialized');
return redirect(route('import.configure', [$job->key]));
}
/**
*
* Show status of import job in JSON.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\JsonResponse
*/
public function json(ImportJob $job)
{
$result = [
$result = [
'started' => false,
'finished' => false,
'running' => false,
'errors' => $job->extended_status['errors'],
'errors' => array_values($job->extended_status['errors']),
'percentage' => 0,
'steps' => $job->extended_status['total_steps'],
'stepsDone' => $job->extended_status['steps_done'],
'steps' => $job->extended_status['steps'],
'done' => $job->extended_status['done'],
'statusText' => trans('firefly.import_status_' . $job->status),
'status' => $job->status,
'finishedText' => '',
];
$percentage = 0;
if ($job->extended_status['total_steps'] !== 0) {
$percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
if ($job->extended_status['steps'] !== 0) {
$result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0);
}
if ($job->status === 'finished') {
// $tagId = $job->extended_status['importTag'];
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
$tag = new Tag;
// $tagId = $job->extended_status['importTag'];
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
$tag = new Tag;
$result['finished'] = true;
$result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]);
}
if ($job->status === 'running') {
$result['started'] = true;
$result['running'] = true;
$result['percentage'] = $percentage;
$result['showPercentage'] = true;
$result['started'] = true;
$result['running'] = true;
}
return Response::json($result);
@ -212,13 +213,12 @@ class ImportController extends Controller
/**
* Step 4. Save the configuration.
*
* @param Request $request
* @param ImportJobRepositoryInterface $repository
* @param ImportJob $job
* @param Request $request
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function postConfigure(Request $request, ImportJobRepositoryInterface $repository, ImportJob $job)
public function postConfigure(Request $request, ImportJob $job)
{
Log::debug('Now in postConfigure()', ['job' => $job->key]);
$configurator = $this->makeConfigurator($job);
@ -237,25 +237,31 @@ class ImportController extends Controller
/**
* @param ImportJob $job
*
* @return string
* @return \Illuminate\Http\JsonResponse
* @throws FireflyException
*/
public function start(ImportJob $job)
{
$routine = new ImportRoutine($job);
$routine->run();
$result = $routine->run();
if ($result) {
return Response::json(['run' => 'ok']);
}
return 'done!';
throw new FireflyException('Job did not complete succesfully.');
}
/**
* This is the last step before the import starts.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function status(ImportJob $job)
{
$statuses = ['configured', 'running', 'finished'];
if (!in_array($job->status, $statuses)) {
return redirect(route('import.configure', [$job->key]));
}
$subTitle = trans('firefly.import_status');
$subTitleIcon = 'fa-star';

View File

@ -103,9 +103,6 @@ class CsvConfigurator implements ConfiguratorInterface
&& $this->job->configuration['column-roles-complete']
&& $this->job->configuration['column-mapping-complete']
) {
$this->job->status = 'configured';
$this->job->save();
return true;
}

View File

@ -1,69 +0,0 @@
<?php
/**
* AccountId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AccountId
*
* @package FireflyIII\Import\Converter
*/
class AccountId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using AssetAccountId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
$account = $repository->find($value);// not mapped? Still try to find it first:
if (!is_null($account->id)) {
$this->setCertainty(90);
Log::debug('Found account by ID ', ['id' => $account->id]);
return $account;
}
$this->setCertainty(0); // should not really happen. If the ID does not match FF, what is FF supposed to do?
return new Account;
}
}

View File

@ -1,87 +0,0 @@
<?php
/**
* AssetAccountIban.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AssetAccountIban
*
* @package FireflyIII\Import\Converter
*/
class AssetAccountIban extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value): Account
{
$value = trim($value);
Log::debug('Going to convert ', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
$this->setCertainty(100);
Log::debug('Found account by ID', ['id' => $account->id]);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByIban($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by IBAN', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => 'Asset account with IBAN ' . $value, 'iban' => $value, 'user' => $this->user->id, 'accountType' => 'asset', 'virtualBalance' => 0,
'active' => true, 'openingBalance' => 0]
);
if (is_null($account->id)) {
$this->setCertainty(0);
Log::info('Could not store new asset account by IBAN', $account->getErrors()->toArray());
return new Account;
}
$this->setCertainty(100);
return $account;
}
}

View File

@ -1,90 +0,0 @@
<?php
/**
* AssetAccountName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AssetAccountName
*
* @package FireflyIII\Import\Converter
*/
class AssetAccountName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using AssetAccountName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByName($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found asset account by name', ['value' => $value, 'id' => $account->id]);
return $account;
}
$account = $repository->store(
['name' => $value, 'iban' => null, 'openingBalance' => 0, 'user' => $this->user->id, 'accountType' => 'asset', 'virtualBalance' => 0,
'active' => true]
);
if (is_null($account->id)) {
$this->setCertainty(0);
Log::info('Could not store new asset account by name', $account->getErrors()->toArray());
return new Account;
}
$this->setCertainty(100);
Log::debug('Created new asset account ', ['name' => $account->name, 'id' => $account->id]);
return $account;
}
}

View File

@ -1,96 +0,0 @@
<?php
/**
* AssetAccountNumber.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AssetAccountNumber
*
* @package FireflyIII\Import\Converter
*/
class AssetAccountNumber extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using AssetAccountNumber', ['value' => $value]);
if (strlen($value) === 0) {
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByAccountNumber($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
// try to find by the name we would give it:
$accountName = 'Asset account with number ' . e($value);
$account = $repository->findByName($accountName, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => $accountName, 'openingBalance' => 0, 'iban' => null, 'user' => $this->user->id,
'accountType' => 'asset',
'virtualBalance' => 0, 'accountNumber' => $value, 'active' => true]
);
if (is_null($account->id)) {
$this->setCertainty(0);
Log::info('Could not store new asset account by account number', $account->getErrors()->toArray());
return new Account;
}
$this->setCertainty(100);
return $account;
}
}

View File

@ -1,76 +0,0 @@
<?php
/**
* BillId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Bill;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use Log;
/**
* Class BillId
*
* @package FireflyIII\Import\Converter
*/
class BillId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Bill
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using BillId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Bill;
}
/** @var BillRepositoryInterface $repository */
$repository = app(BillRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found bill in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$bill = $repository->find(intval($this->mapping[$value]));
if (!is_null($bill->id)) {
Log::debug('Found bill by ID', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
}
// not mapped? Still try to find it first:
$bill = $repository->find($value);
if (!is_null($bill->id)) {
Log::debug('Found bill by ID ', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
// should not really happen. If the ID does not match FF, what is FF supposed to do?
Log::info(sprintf('Could not find bill with ID %d. Will return NULL', $value));
$this->setCertainty(0);
return new Bill;
}
}

View File

@ -1,99 +0,0 @@
<?php
/**
* BillName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Bill;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use Log;
/**
* Class BillName
*
* @package FireflyIII\Import\Converter
*/
class BillName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Bill
* @throws FireflyException
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using BillName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Bill;
}
/** @var BillRepositoryInterface $repository */
$repository = app(BillRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found bill in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$bill = $repository->find(intval($this->mapping[$value]));
if (!is_null($bill->id)) {
Log::debug('Found bill by ID', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
}
// not mapped? Still try to find it first:
$bill = $repository->findByName($value);
if (!is_null($bill->id)) {
Log::debug('Found bill by name ', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
// create new bill. Use a lot of made up values.
$bill = $repository->store(
[
'name' => $value,
'match' => $value,
'amount_min' => 1,
'user' => $this->user->id,
'amount_max' => 10,
'date' => date('Ymd'),
'repeat_freq' => 'monthly',
'skip' => 0,
'automatch' => 0,
'active' => 1,
]
);
if (is_null($bill->id)) {
$this->setCertainty(0);
Log::info('Could not store new bill by name', $bill->getErrors()->toArray());
return new Bill;
}
$this->setCertainty(100);
return $bill;
}
}

View File

@ -1,76 +0,0 @@
<?php
/**
* BudgetId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Budget;
use FireflyIII\Repositories\Budget\BudgetRepositoryInterface;
use Log;
/**
* Class BudgetId
*
* @package FireflyIII\Import\Converter
*/
class BudgetId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Budget
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using BudgetId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Budget;
}
/** @var BudgetRepositoryInterface $repository */
$repository = app(BudgetRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found budget in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$budget = $repository->find(intval($this->mapping[$value]));
if (!is_null($budget->id)) {
Log::debug('Found budget by ID', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
}
// not mapped? Still try to find it first:
$budget = $repository->find($value);
if (!is_null($budget->id)) {
Log::debug('Found budget by ID ', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
// should not really happen. If the ID does not match FF, what is FF supposed to do?
$this->setCertainty(0);
Log::info(sprintf('Could not find budget with ID %d. Will return NULL', $value));
return new Budget;
}
}

View File

@ -1,80 +0,0 @@
<?php
/**
* BudgetName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Budget;
use FireflyIII\Repositories\Budget\BudgetRepositoryInterface;
use Log;
/**
* Class BudgetName
*
* @package FireflyIII\Import\Converter
*/
class BudgetName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Budget
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using BudgetName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Budget;
}
/** @var BudgetRepositoryInterface $repository */
$repository = app(BudgetRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found budget in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$budget = $repository->find(intval($this->mapping[$value]));
if (!is_null($budget->id)) {
Log::debug('Found budget by ID', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
}
// not mapped? Still try to find it first:
$budget = $repository->findByName($value);
if (!is_null($budget->id)) {
Log::debug('Found budget by name ', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
// create new budget. Use a lot of made up values.
$budget = $repository->store(
[
'name' => $value,
'user' => $this->user->id,
]
);
$this->setCertainty(100);
return $budget;
}
}

View File

@ -1,76 +0,0 @@
<?php
/**
* CategoryId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Category;
use FireflyIII\Repositories\Category\CategoryRepositoryInterface;
use Log;
/**
* Class CategoryId
*
* @package FireflyIII\Import\Converter
*/
class CategoryId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Category
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using CategoryId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Category;
}
/** @var CategoryRepositoryInterface $repository */
$repository = app(CategoryRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found category in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$category = $repository->find(intval($this->mapping[$value]));
if (!is_null($category->id)) {
Log::debug('Found category by ID', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
}
// not mapped? Still try to find it first:
$category = $repository->find($value);
if (!is_null($category->id)) {
Log::debug('Found category by ID ', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
// should not really happen. If the ID does not match FF, what is FF supposed to do?
$this->setCertainty(0);
Log::info(sprintf('Could not find category with ID %d. Will return NULL', $value));
return new Category;
}
}

View File

@ -1,80 +0,0 @@
<?php
/**
* CategoryName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Category;
use FireflyIII\Repositories\Category\CategoryRepositoryInterface;
use Log;
/**
* Class CategoryName
*
* @package FireflyIII\Import\Converter
*/
class CategoryName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Category
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using CategoryName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Category;
}
/** @var CategoryRepositoryInterface $repository */
$repository = app(CategoryRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found category in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$category = $repository->find(intval($this->mapping[$value]));
if (!is_null($category->id)) {
Log::debug('Found category by ID', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
}
// not mapped? Still try to find it first:
$category = $repository->findByName($value);
if (!is_null($category->id)) {
Log::debug('Found category by name ', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
// create new category. Use a lot of made up values.
$category = $repository->store(
[
'name' => $value,
'user' => $this->user->id,
]
);
$this->setCertainty(100);
return $category;
}
}

View File

@ -1,71 +0,0 @@
<?php
/**
* CurrencyCode.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencyCode
*
* @package FireflyIII\Import\Converter
*/
class CurrencyCode extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value): TransactionCurrency
{
Log::debug('Going to convert currency code', ['value' => $value]);
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->findByCode($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by code', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
$currency = $repository->store(
[
'name' => $value,
'code' => $value,
'symbol' => $value,
]
);
$this->setCertainty(100);
return $currency;
}
}

View File

@ -1,75 +0,0 @@
<?php
/**
* CurrencyId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencyId
*
* @package FireflyIII\Import\Converter
*/
class CurrencyId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using CurrencyId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new TransactionCurrency;
}
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->find($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by ID ', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
$this->setCertainty(0);
// should not really happen. If the ID does not match FF, what is FF supposed to do?
Log::info(sprintf('Could not find category with ID %d. Will return NULL', $value));
return new TransactionCurrency;
}
}

View File

@ -1,81 +0,0 @@
<?php
/**
* CurrencyName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencyName
*
* @package FireflyIII\Import\Converter
*/
class CurrencyName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using CurrencyName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new TransactionCurrency;
}
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->findByName($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by name ', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
// create new currency
$currency = $repository->store(
[
'name' => $value,
'code' => strtoupper(substr($value, 0, 3)),
'symbol' => strtoupper(substr($value, 0, 1)),
]
);
$this->setCertainty(100);
return $currency;
}
}

View File

@ -1,81 +0,0 @@
<?php
/**
* CurrencySymbol.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencySymbol
*
* @package FireflyIII\Import\Converter
*/
class CurrencySymbol extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using CurrencySymbol', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new TransactionCurrency;
}
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->findBySymbol($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by symbol ', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
// create new currency
$currency = $repository->store(
[
'name' => 'Currency ' . $value,
'code' => $value,
'symbol' => $value,
]
);
$this->setCertainty(100);
return $currency;
}
}

View File

@ -1,53 +0,0 @@
<?php
/**
* Date.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use Carbon\Carbon;
use FireflyIII\Exceptions\FireflyException;
use InvalidArgumentException;
use Log;
/**
* Class Date
*
* @package FireflyIII\Import\Converter
*/
class Date extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Carbon
* @throws FireflyException
*/
public function convert($value): Carbon
{
Log::debug('Going to convert date', ['value' => $value]);
Log::debug('Format: ', ['format' => $this->config['date-format']]);
try {
$date = Carbon::createFromFormat($this->config['date-format'], $value);
} catch (InvalidArgumentException $e) {
Log::info($e->getMessage());
Log::info('Cannot convert this string using the given format.', ['value' => $value, 'format' => $this->config['date-format']]);
$this->setCertainty(0);
return new Carbon;
}
Log::debug('Converted date', ['converted' => $date->toAtomString()]);
$this->setCertainty(100);
return $date;
}
}

View File

@ -1,39 +0,0 @@
<?php
/**
* Description.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
/**
* Class Description
*
* @package FireflyIII\Import\Converter
*/
class Description extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return string
*/
public function convert($value): string
{
// this should replace all control characters
// but leave utf8 intact:
$value = preg_replace('/[\x00-\x1F\x80-\x9F]/u', '', $value);
$this->setCertainty(100);
return strval($value);
}
}

View File

@ -1,39 +0,0 @@
<?php
/**
* ExternalId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
/**
* Class ExternalId
*
* @package FireflyIII\Import\Converter
*/
class ExternalId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return string
*/
public function convert($value): string
{
// this should replace all control characters
// but leave utf8 intact:
$value = preg_replace('/[\x00-\x1F\x80-\x9F]/u', '', $value);
$this->setCertainty(100);
return strval(trim($value));
}
}

View File

@ -1,34 +0,0 @@
<?php
/**
* Ignore.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
/**
* Class Ignore
*
* @package FireflyIII\Import\Converter
*/
class Ignore extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return null
*/
public function convert($value)
{
return null;
}
}

View File

@ -1,91 +0,0 @@
<?php
/**
* OpposingAccountIban.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Account;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class OpposingAccountIban
*
* @package FireflyIII\Import\Converter
*/
class OpposingAccountIban extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value): Account
{
$value = trim($value);
Log::debug('Going to convert opposing IBAN', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByIban($value, []);
if (!is_null($account->id)) {
Log::debug('Found account by IBAN', ['id' => $account->id]);
Log::info(
'The match between IBAN and account is uncertain because the type of transactions may not have been determined.',
['id' => $account->id, 'iban' => $value]
);
$this->setCertainty(50);
return $account;
}
// the IBAN given may not be a valid IBAN. If not, we cannot store by
// iban and we have no opposing account. There should be some kind of fall back
// routine.
try {
$account = $repository->store(
['name' => $value, 'iban' => $value, 'user' => $this->user->id, 'accountType' => 'import', 'virtualBalance' => 0, 'active' => true,
'openingBalance' => 0]
);
$this->setCertainty(100);
} catch (FireflyException $e) {
Log::error($e);
$account = new Account;
}
return $account;
}
}

View File

@ -1,89 +0,0 @@
<?php
/**
* OpposingAccountName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class OpposingAccountName
*
* @package FireflyIII\Import\Converter
*/
class OpposingAccountName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value): Account
{
$value = trim($value);
Log::debug('Going to convert opposing account name', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByName($value, []);
if (!is_null($account->id)) {
Log::debug('Found opposing account by name', ['id' => $account->id]);
Log::info(
'The match between name and account is uncertain because the type of transactions may not have been determined.',
['id' => $account->id, 'name' => $value]
);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => $value, 'iban' => null, 'user' => $this->user->id, 'accountType' => 'import', 'virtualBalance' => 0, 'active' => true,
'openingBalance' => 0,
]
);
if (is_null($account->id)) {
$this->setCertainty(0);
return new Account;
}
$this->setCertainty(100);
Log::debug('Created new opposing account ', ['name' => $account->name, 'id' => $account->id]);
return $account;
}
}

View File

@ -1,91 +0,0 @@
<?php
/**
* OpposingAccountNumber.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class OpposingAccountNumber
*
* @package FireflyIII\Import\Converter
*/
class OpposingAccountNumber extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using OpposingAccountNumber', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByAccountNumber($value, []);
if (!is_null($account->id)) {
Log::debug('Found account by number', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
// try to find by the name we would give it:
$accountName = 'Import account with number ' . e($value);
$account = $repository->findByName($accountName, [AccountType::IMPORT]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => $accountName, 'openingBalance' => 0, 'iban' => null, 'user' => $this->user->id,
'accountType' => 'import',
'virtualBalance' => 0, 'accountNumber' => $value, 'active' => true]
);
$this->setCertainty(100);
return $account;
}
}

View File

@ -67,25 +67,28 @@ class CsvProcessor implements FileProcessorInterface
Log::debug('Now in CsvProcessor run(). Job is now running...');
$entries = $this->getImportArray();
$count = 0;
$index = 0;
Log::notice('Building importable objects from CSV file.');
foreach ($entries as $index => $row) {
// verify if not exists already:
if ($this->hashPresent($row)) {
Log::info(sprintf('Row #%d has already been imported.', $index));
if ($this->rowAlreadyImported($row)) {
$message = sprintf('Row #%d has already been imported.', $index);
$this->job->addError($index, $message);
$this->job->addStepsDone(5); // all steps.
Log::info($message);
continue;
}
$this->objects->push($this->importRow($index, $row));
/**
* 1. Build import entry.
* 2. Validate import entry.
* 3. Store journal.
* 4. Run rules.
*/
$this->job->addStepsDone(1);
$count++;
sleep(1);
}
// if job has no step count, set it now:
$extended = $this->job->extended_status;
if ($extended['steps'] === 0) {
$extended['steps'] = $index * 5;
$this->job->extended_status = $extended;
$this->job->save();
}
return true;
}
@ -148,36 +151,6 @@ class CsvProcessor implements FileProcessorInterface
return $results;
}
/**
* Checks if the row has not been imported before.
*
* TODO for debugging, will always return false.
*
* @param array $array
*
* @noinspection PhpUnreachableStatementInspection
* @return bool
*/
private function hashPresent(array $array): bool
{
$string = json_encode($array);
$hash = hash('sha256', json_encode($string));
$json = json_encode($hash);
$entry = TransactionJournalMeta::
leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('data', $json)
->where('name', 'importHash')
->first();
return false;
if (!is_null($entry)) {
return true;
}
return false;
}
/**
* Take a row, build import journal by annotating each value and storing it in the import journal.
*
@ -207,6 +180,33 @@ class CsvProcessor implements FileProcessorInterface
return $journal;
}
/**
* Checks if the row has not been imported before.
*
* @param array $array
*
* @return bool
*/
private function rowAlreadyImported(array $array): bool
{
$string = json_encode($array);
$hash = hash('sha256', json_encode($string));
$json = json_encode($hash);
$entry = TransactionJournalMeta::
leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('data', $json)
->where('name', 'importHash')
->first();
return rand(1, 10) === 3;
if (!is_null($entry)) {
return true;
}
return false;
}
/**
* And this is the point where the specifix go to work.
*

View File

@ -40,6 +40,7 @@ class ImportRoutine
$this->job = $job;
$this->journals = new Collection;
$this->errors = new Collection;
Log::debug(sprintf('Job ID is #%d', $job->id));
}
/**
@ -48,7 +49,7 @@ class ImportRoutine
public function run(): bool
{
if ($this->job->status !== 'configured') {
Log::error(sprintf('Job %s is in state %s so it cannot be started.', $this->job->key, $this->job->status));
Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->job->status));
return false;
}

View File

@ -58,8 +58,6 @@ class ImportStorage
$this->objects = new Collection;
$this->journals = new Collection;
$this->errors = new Collection;
$this->rules = $this->getUserRules();
}
/**
@ -75,7 +73,8 @@ class ImportStorage
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
$this->job = $job;
$this->rules = $this->getUserRules();
}
/**
@ -100,6 +99,7 @@ class ImportStorage
* @var ImportJournal $object
*/
foreach ($this->objects as $index => $object) {
sleep(4);
Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $object->description));
$errors = new MessageBag;
@ -148,11 +148,20 @@ class ImportStorage
$journal->order = 0;
$journal->tag_count = 0;
$journal->encrypted = 0;
$journal->completed = 0;
$journal->completed = false;
if (rand(1, 10) === 3) {
$journal->date = null;
$journal->description = null;
}
if (!$journal->save()) {
$errorText = join(', ', $journal->getErrors()->all());
$errors->add('no-key', sprintf('Error storing journal: %s', $errorText));
$this->addErrorToJob($index, sprintf('Error storing line #%d: %s', $index, $errorText));
Log::error(sprintf('Could not store line #%d: %s', $index, $errorText));
// add the rest of the steps:
$this->job->addStepsDone(3);
continue;
}
$journal->setMeta('importHash', $object->hash);
@ -184,7 +193,6 @@ class ImportStorage
Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id));
$this->job->addStepsDone(1);
sleep(1);
// category
$category = $object->category->getCategory();
@ -222,6 +230,11 @@ class ImportStorage
if (strlen($object->notes) > 0) {
$journal->setMeta('notes', $object->notes);
}
// set journal completed:
$journal->completed = true;
$journal->save();
$this->job->addStepsDone(1);
// run rules:
@ -230,9 +243,6 @@ class ImportStorage
$this->journals->push($journal);
$this->errors->push($errors);
sleep(1);
}
@ -262,6 +272,18 @@ class ImportStorage
return true;
}
/**
* @param int $index
* @param string $error
*/
private function addErrorToJob(int $index, string $error)
{
$extended = $this->job->extended_status;
$extended['errors'][$index][] = $error;
$this->job->extended_status = $extended;
$this->job->save();
}
/**
* @return Collection
*/

View File

@ -66,13 +66,28 @@ class ImportJob extends Model
throw new NotFoundHttpException;
}
/**
* @param int $index
* @param string $message
*
* @return bool
*/
public function addError(int $index, string $message): bool
{
$extended = $this->extended_status;
$extended['errors'][$index][] = $message;
$this->extended_status = $extended;
return true;
}
/**
* @param int $count
*/
public function addStepsDone(int $count)
{
$status = $this->extended_status;
$status['steps_done'] += $count;
$status = $this->extended_status;
$status['done'] += $count;
$this->extended_status = $status;
$this->save();
@ -84,7 +99,7 @@ class ImportJob extends Model
public function addTotalSteps(int $count)
{
$status = $this->extended_status;
$status['total_steps'] += $count;
$status['steps'] += $count;
$this->extended_status = $status;
$this->save();
@ -165,7 +180,7 @@ class ImportJob extends Model
$disk = Storage::disk('upload');
$encryptedContent = $disk->get($fileName);
$content = Crypt::decrypt($encryptedContent);
Log::debug(sprintf('Content size is %d bytes.', $content));
Log::debug(sprintf('Content size is %d bytes.', strlen($content)));
return $content;
}

View File

@ -58,12 +58,12 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'new';
$importJob->configuration = [];
$importJob->extended_status = [
'total_steps' => 0,
'steps_done' => 0,
'import_count' => 0,
'importTag' => 0,
'errors' => [],
'steps' => 0,
'done' => 0,
'importTag' => 0,
'errors' => [],
];
$importJob->save();
@ -157,8 +157,6 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$disk->put($newName, $contentEncrypted);
Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]);
}
$job->status = 'initialized';
$job->save();
return true;
}

View File

@ -95,7 +95,7 @@ class Map implements ConfigurationInterface
// save number of rows, thus number of steps, in job:
$steps = $rowIndex * 5;
$extended = $this->job->extended_status;
$extended['total_steps'] = $steps;
$extended['steps'] = $steps;
$this->job->extended_status = $extended;
$this->job->save();

View File

@ -10,153 +10,175 @@
/** global: jobImportUrl, langImportSingleError, langImportMultiError, jobStartUrl, langImportTimeOutError, langImportFinished, langImportFatalError */
var displayStatus = 'initial';
var timeOutId;
// var startedImport = false;
var startInterval = 2000;
var startInterval = 1000;
var interval = 500;
// var timeoutLimit = 5000;
// var currentLimit = 0;
// var stepCount = 0;
// after this many tries, stop checking when the job is not running anyway.
var maxNotRunningCount = 20;
var notRunningCount = 0;
// these vars are used to detect a stalled job:
var numberOfSteps = 0;
var numberOfReports = 0;
var jobFailed = false;
// counts how many errors have been detected
var knownErrors = 0;
$(function () {
"use strict";
//$('#import-status-intro').hide();
//$('#import-status-more-info').hide();
// check status, every 500 ms.
timeOutId = setTimeout(checkImportStatus, startInterval);
// button to start import routine:
$('.start-job').click(startJob);
});
function startJob() {
console.log('Job started.');
$.post(jobStartUrl);
/**
* Downloads some JSON and responds to its content to see what the status is of the current import.
*/
function checkImportStatus() {
console.log('checkImportStatus()');
$.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport);
}
// reset not running thing
notRunningCount = 0;
// check status, every 500 ms.
timeOutId = setTimeout(checkImportStatus, startInterval);
/**
* This method is called when the JSON query returns an error. If possible, this error is relayed to the user.
*/
function failedJobImport(jqxhr, textStatus, error) {
console.log('failedJobImport()');
console.log(textStatus);
console.log(error);
// hide all possible boxes:
$('.statusbox').hide();
// fill in some details:
var errorMessage = textStatus + " " + error;
$('.import_error_txt').text(errorMessage);
// show the error box:
$('.errorbox').show();
}
/**
* This method is called when the job enquiry (JSON) returns some info.
* It also decides whether or not to check again.
*
* @param data
*/
function reportOnJobImport(data) {
console.log('reportOnJobImport()');
console.log('Job status is: "' + data.status + '".');
switch (data.status) {
case "configured":
// job is ready. Do not check again, just show the start-box. Hide the rest.
$('.statusbox').hide();
$('.status_configured').show();
break;
case "running":
// job is running! Show the running box:
$('.statusbox').hide();
$('.status_running').show();
// update the bar
updateBar(data);
// update the status text:
updateStatusText(data);
// report on detected errors:
reportOnErrors(data);
if (jobIsStalled(data)) {
// do something
showStalledBox();
} else {
// check again in 500ms
timeOutId = setTimeout(checkImportStatus, interval);
}
break;
case "finished":
$('.statusbox').hide();
$('.status_finished').show();
// show text:
break;
}
}
/**
* Shows an error when the job seems to be stalled.
*/
function showStalledBox() {
$('.statusbox').hide();
$('.errorbox').show();
$('.import_error_txt').text(langImportTimeOutError);
}
/**
* Detects if a job is frozen.
*
* @param data
*/
function jobIsStalled(data) {
console.log('jobIsStalled(' + numberOfSteps + ', ' + numberOfReports + ')');
if (data.steps === numberOfSteps) {
numberOfReports++;
}
if (data.done !== numberOfSteps) {
numberOfReports = 0;
}
if (numberOfReports > 20) {
return true;
}
numberOfSteps = data.done;
return false;
}
function checkImportStatus() {
"use strict";
$.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport);
/**
* This function tells Firefly start the job. It will also initialize a re-check in 500ms time.
*/
function startJob() {
// disable the button, add loading thing.
$('.start-job').prop('disabled', true).text('...');
console.log('startJob()');
$.post(jobStartUrl).fail(reportOnSubmitError);
// check status, every 500 ms.
timeOutId = setTimeout(checkImportStatus, startInterval);
}
function reportToConsole(data) {
console.log('status: ' + data.status + ', steps: ' + data.steps + ', stepsDone: ' + data.stepsDone);
function reportOnSubmitError() {
// stop the refresh thing
console.error('Clear time out');
clearTimeout(timeOutId);
//console.log('more status: ' + data);
}
// hide all possible boxes:
$('.statusbox').hide();
function reportOnJobImport(data) {
"use strict";
if (data.running == false) {
notRunningCount++;
}
// fill in some details:
var errorMessage = "Time out.";
displayCorrectBox(data.status);
reportToConsole(data);
updateBar(data);
//reportErrors(data);
reportStatus(data);
//updateTimeout(data);
$('.import_error_txt').text(errorMessage);
//if (importJobFinished(data)) {
// finishedJob(data);
// return;
//}
// same number of steps as last time?
//if (currentLimit > timeoutLimit) {
// timeoutError();
// return;
//}
// if the job has not actually started, do so now:
//if (!data.started && !startedImport) {
// kickStartJob();
// return;
//}
// trigger another check.
if (notRunningCount < maxNotRunningCount && data.finished === false) {
timeOutId = setTimeout(checkImportStatus, interval);
}
if (notRunningCount >= maxNotRunningCount && data.finished === false) {
console.error('Job still not running, stop checking for it.');
}
if(data.finished === true) {
console.log('Job is done');
}
// show the error box:
$('.errorbox').show();
jobFailed = true;
}
function displayCorrectBox(status) {
console.log('Current job state is ' + status);
if (status === 'configured' && displayStatus === 'initial') {
// hide some boxes:
$('.status_initial').hide();
return;
}
if (status === 'running') {
// hide some boxes:
$('.status_initial').hide();
$('.status_running').show();
$('.status_configured').hide();
return;
}
if(status === 'finished') {
$('.status_initial').hide();
$('.status_running').hide();
$('.status_configured').hide();
$('.status_finished').show();
}
console.error('CANNOT HANDLE CURRENT STATE');
}
function importComplete() {
"use strict";
var bar = $('#import-status-bar');
bar.removeClass('active');
}
/**
* This method updates the percentage bar thing if the job is running!
*/
function updateBar(data) {
"use strict";
console.log('updateBar()');
var bar = $('#import-status-bar');
if (data.showPercentage) {
if (data.percentage > 0) {
console.log('Going to update bar with percentage.');
bar.addClass('progress-bar-success').removeClass('progress-bar-info');
bar.attr('aria-valuenow', data.percentage);
bar.css('width', data.percentage + '%');
$('#import-status-bar').text(data.stepsDone + '/' + data.steps);
if (data.percentage >= 100) {
importComplete();
return;
}
return;
$('#import-status-bar').text(data.done + '/' + data.steps);
return true;
}
console.log('Going to update bar without percentage.');
// dont show percentage:
@ -164,9 +186,223 @@ function updateBar(data) {
bar.attr('aria-valuenow', 100);
bar.css('width', '100%');
}
/**
* Add text with current import status.
* @param data
*/
function updateStatusText(data) {
"use strict";
console.log('Going to report status: ' + data.statusText);
$('#import-status-txt').removeClass('text-danger').text(data.statusText);
}
/**
* Report on errors found in import:
* @param data
*/
function reportOnErrors(data) {
console.log('reportOnErrors()')
if (knownErrors === data.errors.length) {
console.log(knownErrors + ' = ' + data.errors.length);
return;
}
if (data.errors.length === 0) {
return;
}
if (data.errors.length === 1) {
$('#import-status-error-intro').text(langImportSingleError);
//'An error has occured during the import. The import can continue, however.'
}
if (data.errors.length > 1) {
// 'Errors have occured during the import. The import can continue, however.'
$('#import-status-error-intro').text(langImportMultiError);
}
$('.info_errors').show();
// fill the list with error texts
$('#import-status-error-list').empty();
for (var i = 0; i < data.errors.length; i++) {
var errorSet = data.errors[i];
for (var j = 0; j < errorSet.length; j++) {
var item = $('<li>').html(errorSet[j]);
$('#import-status-error-list').append(item);
}
}
return;
}
//
// var displayStatus = 'initial';
//
//
//
// // var startedImport = false;
//
// // var timeoutLimit = 5000;
// // var currentLimit = 0;
// // var stepCount = 0;
//
// // count the number of errors so we have an idea if the list must be recreated.
// var errorCount = 0;
//
// // after this many tries, stop checking when the job is not running anyway.
// var maxNotRunningCount = 20;
// var notRunningCount = 0;
//
// $(function () {
// "use strict";
//
// //$('#import-status-intro').hide();
// //$('#import-status-more-info').hide();
//
// // check status, every 500 ms.
// //timeOutId = setTimeout(checkImportStatus, startInterval);
//
// // button to start import routine:
// $('.start-job').click(startJob);
//
// });
//
// function startJob() {
// console.log('Job started.');
// $.post(jobStartUrl);
//
// // reset not running thing
// notRunningCount = 0;
// // check status, every 500 ms.
// timeOutId = setTimeout(checkImportStatus, startInterval);
//
// return false;
// }
//
// function checkImportStatus() {
// "use strict";
// $.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport);
// }
//
// function reportToConsole(data) {
// console.log('status: ' + data.status + ', steps: ' + data.steps + ', done: ' + data.done);
//
// //console.log('more status: ' + data);
// }
//
// function reportOnJobImport(data) {
// "use strict";
// if (data.running == false) {
// notRunningCount++;
// }
//
// displayCorrectBox(data.status);
// reportToConsole(data);
// updateBar(data);
// reportErrors(data);
// reportStatus(data);
// //updateTimeout(data);
//
// //if (importJobFinished(data)) {
// // finishedJob(data);
// // return;
// //}
//
//
// // same number of steps as last time?
// //if (currentLimit > timeoutLimit) {
// // timeoutError();
// // return;
// //}
//
// // if the job has not actually started, do so now:
// //if (!data.started && !startedImport) {
// // kickStartJob();
// // return;
// //}
//
// // trigger another check.
// if (notRunningCount < maxNotRunningCount && data.finished === false) {
// timeOutId = setTimeout(checkImportStatus, interval);
// }
// if (notRunningCount >= maxNotRunningCount && data.finished === false) {
// console.error('Job still not running, stop checking for it.');
// }
// if (data.finished === true) {
// console.log('Job is done');
// }
//
// }
//
// function displayCorrectBox(status) {
// console.log('Current job state is ' + status);
// if (status === 'configured' && displayStatus === 'initial') {
// // hide some boxes:
// $('.status_initial').hide();
// return;
// }
// if (status === 'running') {
// // hide some boxes:
// $('.status_initial').hide();
// $('.status_running').show();
// $('.status_configured').hide();
//
//
// return;
// }
//
// if (status === 'finished') {
// $('.status_initial').hide();
// $('.status_running').hide();
// $('.status_configured').hide();
// $('.status_finished').show();
// }
//
//
// console.error('CANNOT HANDLE CURRENT STATE');
// }
//
//
// function importComplete() {
// "use strict";
// var bar = $('#import-status-bar');
// bar.removeClass('active');
// }
//
// function updateBar(data) {
// "use strict";
//
// var bar = $('#import-status-bar');
// if (data.showPercentage) {
// console.log('Going to update bar with percentage.');
// bar.addClass('progress-bar-success').removeClass('progress-bar-info');
// bar.attr('aria-valuenow', data.percentage);
// bar.css('width', data.percentage + '%');
// $('#import-status-bar').text(data.done + '/' + data.steps);
//
// if (data.percentage >= 100) {
// importComplete();
// return;
// }
// return;
// }
// console.log('Going to update bar without percentage.');
// // dont show percentage:
// bar.removeClass('progress-bar-success').addClass('progress-bar-info');
// bar.attr('aria-valuenow', 100);
// bar.css('width', '100%');
// }
// //
// function reportErrors(data) {
// "use strict";
// console.log('Will now reportErrors() with ' + data.errors.length + ' errors.');
// if (data.errors.length < 1) {
// return;
// }
// $('.info_errors').show();
// if (data.errors.length === errorCount) {
// console.log('Error count is the same as before, do not response.');
// }
// errorCount = data.errors.length;
// if (data.errors.length === 1) {
// $('#import-status-error-intro').text(langImportSingleError);
// //'An error has occured during the import. The import can continue, however.'
@ -179,78 +415,81 @@ function updateBar(data) {
// // fill the list with error texts
// $('#import-status-error-list').empty();
// for (var i = 0; i < data.errors.length; i++) {
// var item = $('<li>').html(data.errors[i]);
// $('#import-status-error-list').append(item);
// var errorSet = data.errors[i];
// for (var j = 0; j < errorSet.length; j++) {
// var item = $('<li>').html(errorSet[j]);
// $('#import-status-error-list').append(item);
// }
// }
// }
//
function reportStatus(data) {
"use strict";
console.log('Going to report status: ' + data.statusText);
$('#import-status-txt').removeClass('text-danger').text(data.statusText);
}
//
// function kickStartJob() {
// //
// function reportStatus(data) {
// "use strict";
// $.post(jobStartUrl);
// startedTheImport();
// startedImport = true;
// console.log('Going to report status: ' + data.statusText);
// $('#import-status-txt').removeClass('text-danger').text(data.statusText);
// }
// //
// // function kickStartJob() {
// // "use strict";
// // $.post(jobStartUrl);
// // startedTheImport();
// // startedImport = true;
// // }
// //
// // function updateTimeout(data) {
// // "use strict";
// // if (data.done !== stepCount) {
// // stepCount = data.done;
// // currentLimit = 0;
// // return;
// // }
// //
// // currentLimit = currentLimit + interval;
// // }
// //
// // function timeoutError() {
// // "use strict";
// // // set status
// // $('#import-status-txt').addClass('text-danger').text(langImportTimeOutError);
// //
// // // remove progress bar.
// // $('#import-status-holder').hide();
// //
// // }
// //
// // function importJobFinished(data) {
// // "use strict";
// // return data.finished;
// // }
// //
// // function finishedJob(data) {
// // "use strict";
// // // "There was an error during the import routine. Please check the log files. The error seems to be: '"
// // $('#import-status-txt').removeClass('text-danger').addClass('text-success').text(langImportFinished);
// //
// // // remove progress bar.
// // $('#import-status-holder').hide();
// //
// // // show info:
// // $('#import-status-intro').show();
// // $('#import-status-more-info').html(data.finishedText).show();
// //
// // }
// //
// //
// //
// // function startedTheImport() {
// // "use strict";
// // setTimeout(checkImportStatus, interval);
// // }
//
// function updateTimeout(data) {
// "use strict";
// if (data.stepsDone !== stepCount) {
// stepCount = data.stepsDone;
// currentLimit = 0;
// return;
// }
//
// currentLimit = currentLimit + interval;
// }
//
// function timeoutError() {
// function failedJobImport(jqxhr, textStatus, error) {
// "use strict";
// console.error('Job status failed!');
// // set status
// $('#import-status-txt').addClass('text-danger').text(langImportTimeOutError);
//
// // remove progress bar.
// $('#import-status-holder').hide();
//
// }
//
// function importJobFinished(data) {
// "use strict";
// return data.finished;
// }
//
// function finishedJob(data) {
// "use strict";
// // "There was an error during the import routine. Please check the log files. The error seems to be: '"
// $('#import-status-txt').removeClass('text-danger').addClass('text-success').text(langImportFinished);
// $('#import-status-txt').addClass('text-danger').text(langImportFatalError + ' ' + textStatus + ' ' + error);
//
// // remove progress bar.
// $('#import-status-holder').hide();
//
// // show info:
// $('#import-status-intro').show();
// $('#import-status-more-info').html(data.finishedText).show();
//
// }
//
//
//
// function startedTheImport() {
// "use strict";
// setTimeout(checkImportStatus, interval);
// }
function failedJobImport(jqxhr, textStatus, error) {
"use strict";
console.error('Job status failed!');
// set status
// "There was an error during the import routine. Please check the log files. The error seems to be: '"
$('#import-status-txt').addClass('text-danger').text(langImportFatalError + ' ' + textStatus + ' ' + error);
// remove progress bar.
$('#import-status-holder').hide();
}
// }

View File

@ -987,7 +987,7 @@ return [
'import_file_help' => 'Select your file',
'import_status_settings_complete' => 'The import is ready to start.',
'import_status_import_complete' => 'The import has completed.',
'import_status_running' => 'The import is currently running. Please be patient.',
'import_status_running' => 'The import is currently running. Please be patient.',
'import_status_header' => 'Import status and progress',
'import_status_errors' => 'Import errors',
'import_status_report' => 'Import report',
@ -1009,6 +1009,10 @@ return [
'import_finished_intro' => 'The import has finished! You can now see the new transactions in Firefly.',
'import_finished_text_without_link' => 'It seems there is no tag that points to all your imported transactions. Please look for your imported data in the menu on the left, under "Transactions".',
'import_finished_text_with_link' => 'You can find a list of your imported transactions on the page of the <a href="tags/show/:tag">tag that was created for this import</a>.',
'please_wait_for_status' => 'Please wait for Firefly III to check on your import',
'box_will_refresh' => 'This box will auto-refresh...',
'import_job_status' => 'Import routine status..',
'see_help_top_right' => 'Welcome to Firefly\'s import routine. Please check out the help pages in the top right corner.',
// sandstorm.io errors and messages:
'sandstorm_not_available' => 'This function is not available when you are using Firefly III within a Sandstorm.io environment.',

View File

@ -7,20 +7,45 @@
{# Initial display. Will refresh (and disappear almost immediately. #}
<div class="row status_initial" style="display:none;">
<div class="row status_initial statusbox">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-primary">
<div class="box-header with-border">
<h3 class="box-title">Nothing to see here...</h3>
<h3 class="box-title">{{ 'please_wait_for_status'|_ }}</h3>
</div>
<div class="box-body">
<p>This box will be replaced in a moment with the status of your import.</p>
<p>
{{ 'box_will_refresh'|_ }}
</p>
</div>
</div>
</div>
</div>
<div class="row status_configured">
{# Error display. Will be shown (duh) when something goes horribly wrong. #}
<div class="row errorbox" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-danger">
<div class="box-header with-border">
<h3 class="box-title">{{ 'import_error_occurred'|_ }}</h3>
</div>
<div class="box-body">
<p>
{{ 'import_error'|_ }}
</p>
<p class="text-danger import_error_txt">
</p>
<p>
{{ 'import_error_further_instructions'|_ }}
</p>
</div>
</div>
</div>
</div>
{# Box for when the job is ready to start #}
<div class="row status_configured statusbox" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-primary">
<div class="box-header with-border">
@ -39,8 +64,7 @@
class="fa fa-fw fa-download"></i> {{ 'import_download_config'|_ }}</a>
</div>
<div class="col-lg-4">
<a href="#" class="btn btn-success start-job"><i
class="fa fa-fw fa-gears"></i> {{ 'import_start_import'|_ }}</a>
<button class="btn btn-success start-job"><i class="fa fa-fw fa-gears"></i> {{ 'import_start_import'|_ }}</button>
</div>
</div>
<p>
@ -54,28 +78,29 @@
</div>
</div>
<div class="row status_running" style="display: none;">
{# Box for when the job is running! #}
<div class="row status_running statusbox" style="display: none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-primary">
<div class="box-header with-border">
<h3 class="box-title" id="import-status-title">Job status</h3>
<h3 class="box-title" id="import-status-title">{{ 'import_job_status'|_ }}</h3>
</div>
<div class="box-body">
<div id="import-status-holder">
<div class="progress" id="import-status-holder">
<div id="import-status-bar" class="progress-bar progress-bar-info active progress-bar-striped" role="progressbar"
aria-valuenow="100" aria-valuemin="0"
aria-valuemax="100" style="width: 100%">
aria-valuemax="100" style="width: 100%;min-width:40px;">
</div>
</div>
<p id="import-status-txt">{{ 'import_status_settings_complete'|_ }}</p>
<p id="import-status-txt">{{ 'import_status_ready_to_start'|_ }}</p>
</div>
</div>
</div>
</div>
</div>
<div class="row status_finished" style="display:none;">
<div class="row status_finished statusbox" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-default">
<div class="box-header with-border">
@ -91,6 +116,60 @@
</div>
</div>
{# box to show error information. #}
<div class="row info_errors" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-danger">
<div class="box-header with-border">
<h3 class="box-title">{{ 'import_status_errors'|_ }}</h3>
</div>
<div class="box-body">
<p id="import-status-error-intro">
No errors detected.
</p>
<div id="import-status-error-list"></div>
</div>
</div>
</div>
</div>
{#
<div class="row status_finished" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-default">
<div class="box-header with-border">
<h3 class="box-title">{{ 'import_status_report'|_ }}</h3>
</div>
<div class="box-body">
<p id="import-status-intro">
{{ 'import_finished_report'|_ }}
</p>
<p id="import-status-more-info"></p>
</div>
</div>
</div>
</div>
<div class="row info_errors" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-danger">
<div class="box-header with-border">
<h3 class="box-title">{{ 'import_status_errors'|_ }}</h3>
</div>
<div class="box-body">
<p id="import-status-error-intro">
No errors detected.
</p>
<ul id="import-status-error-list"></ul>
</div>
</div>
</div>
</div>
#}
{#