mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2024-12-29 10:21:25 -06:00
More code for import routine.
This commit is contained in:
James Cole
parent
cddaccb7f7
commit
edb355941c
@ -20,6 +20,7 @@ use FireflyIII\Import\ImportProcedureInterface;
|
||||
use FireflyIII\Import\Routine\ImportRoutine;
|
||||
use FireflyIII\Import\Storage\ImportStorage;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Models\Tag;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
|
||||
use FireflyIII\Repositories\User\UserRepositoryInterface;
|
||||
@ -188,11 +189,12 @@ class ImportController extends Controller
|
||||
if ($job->extended_status['total_steps'] !== 0) {
|
||||
$percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
|
||||
}
|
||||
if ($job->status === 'complete') {
|
||||
$tagId = $job->extended_status['importTag'];
|
||||
/** @var TagRepositoryInterface $repository */
|
||||
$repository = app(TagRepositoryInterface::class);
|
||||
$tag = $repository->find($tagId);
|
||||
if ($job->status === 'finished') {
|
||||
// $tagId = $job->extended_status['importTag'];
|
||||
// /** @var TagRepositoryInterface $repository */
|
||||
// $repository = app(TagRepositoryInterface::class);
|
||||
// $tag = $repository->find($tagId);
|
||||
$tag = new Tag;
|
||||
$result['finished'] = true;
|
||||
$result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]);
|
||||
}
|
||||
@ -201,6 +203,7 @@ class ImportController extends Controller
|
||||
$result['started'] = true;
|
||||
$result['running'] = true;
|
||||
$result['percentage'] = $percentage;
|
||||
$result['showPercentage'] = true;
|
||||
}
|
||||
|
||||
return Response::json($result);
|
||||
@ -241,7 +244,7 @@ class ImportController extends Controller
|
||||
$routine = new ImportRoutine($job);
|
||||
$routine->run();
|
||||
|
||||
return 'ok dan';
|
||||
return 'done!';
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -64,8 +64,6 @@ class CsvProcessor implements FileProcessorInterface
|
||||
*/
|
||||
public function run(): bool
|
||||
{
|
||||
$this->job->status = 'running';
|
||||
$this->job->save();
|
||||
Log::debug('Now in CsvProcessor run(). Job is now running...');
|
||||
|
||||
$entries = $this->getImportArray();
|
||||
@ -84,9 +82,9 @@ class CsvProcessor implements FileProcessorInterface
|
||||
* 3. Store journal.
|
||||
* 4. Run rules.
|
||||
*/
|
||||
$this->job->addTotalSteps(4);
|
||||
$this->job->addStepsDone(1);
|
||||
$count++;
|
||||
sleep(1);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@ -21,10 +21,10 @@ use Log;
|
||||
class ImportRoutine
|
||||
{
|
||||
|
||||
/** @var Collection */
|
||||
public $journals;
|
||||
/** @var Collection */
|
||||
public $errors;
|
||||
/** @var Collection */
|
||||
public $journals;
|
||||
/** @var int */
|
||||
public $lines = 0;
|
||||
/** @var ImportJob */
|
||||
@ -39,7 +39,7 @@ class ImportRoutine
|
||||
{
|
||||
$this->job = $job;
|
||||
$this->journals = new Collection;
|
||||
$this->errors = new Collection;
|
||||
$this->errors = new Collection;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -60,8 +60,14 @@ class ImportRoutine
|
||||
/** @var FileProcessorInterface $processor */
|
||||
$processor = app($class);
|
||||
$processor->setJob($this->job);
|
||||
|
||||
set_time_limit(0);
|
||||
if ($this->job->status == 'configured') {
|
||||
|
||||
// set job as "running"...
|
||||
$this->job->status = 'running';
|
||||
$this->job->save();
|
||||
|
||||
Log::debug('Job is configured, start with run()');
|
||||
$processor->run();
|
||||
$objects = $processor->getObjects();
|
||||
@ -81,7 +87,10 @@ class ImportRoutine
|
||||
$this->job->save();
|
||||
|
||||
$this->journals = $storage->journals;
|
||||
$this->errors = $storage->errors;
|
||||
$this->errors = $storage->errors;
|
||||
|
||||
// run rules:
|
||||
|
||||
|
||||
Log::debug(sprintf('Done with import job %s', $this->job->key));
|
||||
|
||||
|
||||
@ -17,10 +17,12 @@ use FireflyIII\Import\Object\ImportJournal;
|
||||
use FireflyIII\Models\Account;
|
||||
use FireflyIII\Models\AccountType;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Models\Rule;
|
||||
use FireflyIII\Models\Transaction;
|
||||
use FireflyIII\Models\TransactionCurrency;
|
||||
use FireflyIII\Models\TransactionJournal;
|
||||
use FireflyIII\Models\TransactionType;
|
||||
use FireflyIII\Rules\Processor;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use Log;
|
||||
@ -45,6 +47,8 @@ class ImportStorage
|
||||
private $job;
|
||||
/** @var Collection */
|
||||
private $objects;
|
||||
/** @var Collection */
|
||||
private $rules;
|
||||
|
||||
/**
|
||||
* ImportStorage constructor.
|
||||
@ -54,6 +58,8 @@ class ImportStorage
|
||||
$this->objects = new Collection;
|
||||
$this->journals = new Collection;
|
||||
$this->errors = new Collection;
|
||||
$this->rules = $this->getUserRules();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@ -80,7 +86,6 @@ class ImportStorage
|
||||
$this->objects = $objects;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Do storage of import objects
|
||||
*/
|
||||
@ -88,6 +93,8 @@ class ImportStorage
|
||||
{
|
||||
$this->defaultCurrency = Amount::getDefaultCurrencyByUser($this->job->user);
|
||||
|
||||
|
||||
// routine below consists of 3 steps.
|
||||
/**
|
||||
* @var int $index
|
||||
* @var ImportJournal $object
|
||||
@ -130,6 +137,8 @@ class ImportStorage
|
||||
$transactionType = TransactionType::whereType(TransactionType::TRANSFER)->first();
|
||||
}
|
||||
|
||||
$this->job->addStepsDone(1);
|
||||
|
||||
$journal = new TransactionJournal;
|
||||
$journal->user_id = $this->job->user_id;
|
||||
$journal->transaction_type_id = $transactionType->id;
|
||||
@ -174,6 +183,9 @@ class ImportStorage
|
||||
}
|
||||
Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id));
|
||||
|
||||
$this->job->addStepsDone(1);
|
||||
sleep(1);
|
||||
|
||||
// category
|
||||
$category = $object->category->getCategory();
|
||||
if (!is_null($category->id)) {
|
||||
@ -210,11 +222,67 @@ class ImportStorage
|
||||
if (strlen($object->notes) > 0) {
|
||||
$journal->setMeta('notes', $object->notes);
|
||||
}
|
||||
$this->job->addStepsDone(1);
|
||||
|
||||
// run rules:
|
||||
$this->applyRules($journal);
|
||||
$this->job->addStepsDone(1);
|
||||
|
||||
$this->journals->push($journal);
|
||||
$this->errors->push($errors);
|
||||
|
||||
|
||||
sleep(1);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param TransactionJournal $journal
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
protected function applyRules(TransactionJournal $journal): bool
|
||||
{
|
||||
if ($this->rules->count() > 0) {
|
||||
|
||||
/** @var Rule $rule */
|
||||
foreach ($this->rules as $rule) {
|
||||
Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id));
|
||||
$processor = Processor::make($rule);
|
||||
$processor->handleTransactionJournal($journal);
|
||||
|
||||
if ($rule->stop_processing) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Collection
|
||||
*/
|
||||
private function getUserRules(): Collection
|
||||
{
|
||||
$set = Rule::distinct()
|
||||
->where('rules.user_id', $this->job->user->id)
|
||||
->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id')
|
||||
->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id')
|
||||
->where('rule_groups.active', 1)
|
||||
->where('rule_triggers.trigger_type', 'user_action')
|
||||
->where('rule_triggers.trigger_value', 'store-journal')
|
||||
->where('rules.active', 1)
|
||||
->orderBy('rule_groups.order', 'ASC')
|
||||
->orderBy('rules.order', 'ASC')
|
||||
->get(['rules.*', 'rule_groups.order']);
|
||||
Log::debug(sprintf('Found %d user rules.', $set->count()));
|
||||
|
||||
return $set;
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -133,6 +133,9 @@ class Tag extends Model
|
||||
*/
|
||||
public function getTagAttribute($value)
|
||||
{
|
||||
if(is_null($value)) {
|
||||
return null;
|
||||
}
|
||||
return Crypt::decrypt($value);
|
||||
}
|
||||
|
||||
|
||||
@ -92,6 +92,12 @@ class Map implements ConfigurationInterface
|
||||
foreach ($this->data as $index => $entry) {
|
||||
$this->data[$index]['values'] = array_unique($this->data[$index]['values']);
|
||||
}
|
||||
// save number of rows, thus number of steps, in job:
|
||||
$steps = $rowIndex * 5;
|
||||
$extended = $this->job->extended_status;
|
||||
$extended['total_steps'] = $steps;
|
||||
$this->job->extended_status = $extended;
|
||||
$this->job->save();
|
||||
|
||||
return $this->data;
|
||||
|
||||
|
||||
@ -14,12 +14,17 @@ var displayStatus = 'initial';
|
||||
var timeOutId;
|
||||
|
||||
|
||||
var startedImport = false;
|
||||
// var startedImport = false;
|
||||
var startInterval = 2000;
|
||||
var interval = 500;
|
||||
var timeoutLimit = 5000;
|
||||
var currentLimit = 0;
|
||||
var stepCount = 0;
|
||||
// var timeoutLimit = 5000;
|
||||
// var currentLimit = 0;
|
||||
// var stepCount = 0;
|
||||
|
||||
// after this many tries, stop checking when the job is not running anyway.
|
||||
var maxNotRunningCount = 20;
|
||||
var notRunningCount = 0;
|
||||
|
||||
$(function () {
|
||||
"use strict";
|
||||
|
||||
@ -37,6 +42,12 @@ $(function () {
|
||||
function startJob() {
|
||||
console.log('Job started.');
|
||||
$.post(jobStartUrl);
|
||||
|
||||
// reset not running thing
|
||||
notRunningCount = 0;
|
||||
// check status, every 500 ms.
|
||||
timeOutId = setTimeout(checkImportStatus, startInterval);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -45,12 +56,23 @@ function checkImportStatus() {
|
||||
$.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport);
|
||||
}
|
||||
|
||||
function reportToConsole(data) {
|
||||
console.log('status: ' + data.status + ', steps: ' + data.steps + ', stepsDone: ' + data.stepsDone);
|
||||
|
||||
//console.log('more status: ' + data);
|
||||
}
|
||||
|
||||
function reportOnJobImport(data) {
|
||||
"use strict";
|
||||
if (data.running == false) {
|
||||
notRunningCount++;
|
||||
}
|
||||
|
||||
displayCorrectBox(data.status);
|
||||
//updateBar(data);
|
||||
reportToConsole(data);
|
||||
updateBar(data);
|
||||
//reportErrors(data);
|
||||
//reportStatus(data);
|
||||
reportStatus(data);
|
||||
//updateTimeout(data);
|
||||
|
||||
//if (importJobFinished(data)) {
|
||||
@ -72,36 +94,47 @@ function reportOnJobImport(data) {
|
||||
//}
|
||||
|
||||
// trigger another check.
|
||||
//timeOutId = setTimeout(checkImportStatus, interval);
|
||||
if (notRunningCount < maxNotRunningCount && data.finished === false) {
|
||||
timeOutId = setTimeout(checkImportStatus, interval);
|
||||
}
|
||||
if (notRunningCount >= maxNotRunningCount && data.finished === false) {
|
||||
console.error('Job still not running, stop checking for it.');
|
||||
}
|
||||
if(data.finished === true) {
|
||||
console.log('Job is done');
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function displayCorrectBox(status) {
|
||||
console.log('Current job state is ' + status);
|
||||
if(status === 'configured' && displayStatus === 'initial') {
|
||||
if (status === 'configured' && displayStatus === 'initial') {
|
||||
// hide some boxes:
|
||||
$('.status_initial').hide();
|
||||
return;
|
||||
}
|
||||
if (status === 'running') {
|
||||
// hide some boxes:
|
||||
$('.status_initial').hide();
|
||||
$('.status_running').show();
|
||||
$('.status_configured').hide();
|
||||
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if(status === 'finished') {
|
||||
$('.status_initial').hide();
|
||||
$('.status_running').hide();
|
||||
$('.status_configured').hide();
|
||||
$('.status_finished').show();
|
||||
}
|
||||
|
||||
|
||||
console.error('CANNOT HANDLE CURRENT STATE');
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
function importComplete() {
|
||||
"use strict";
|
||||
var bar = $('#import-status-bar');
|
||||
@ -110,8 +143,10 @@ function importComplete() {
|
||||
|
||||
function updateBar(data) {
|
||||
"use strict";
|
||||
|
||||
var bar = $('#import-status-bar');
|
||||
if (data.showPercentage) {
|
||||
console.log('Going to update bar with percentage.');
|
||||
bar.addClass('progress-bar-success').removeClass('progress-bar-info');
|
||||
bar.attr('aria-valuenow', data.percentage);
|
||||
bar.css('width', data.percentage + '%');
|
||||
@ -123,92 +158,95 @@ function updateBar(data) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
console.log('Going to update bar without percentage.');
|
||||
// dont show percentage:
|
||||
bar.removeClass('progress-bar-success').addClass('progress-bar-info');
|
||||
bar.attr('aria-valuenow', 100);
|
||||
bar.css('width', '100%');
|
||||
}
|
||||
|
||||
function reportErrors(data) {
|
||||
"use strict";
|
||||
if (data.errors.length === 1) {
|
||||
$('#import-status-error-intro').text(langImportSingleError);
|
||||
//'An error has occured during the import. The import can continue, however.'
|
||||
}
|
||||
if (data.errors.length > 1) {
|
||||
// 'Errors have occured during the import. The import can continue, however.'
|
||||
$('#import-status-error-intro').text(langImportMultiError);
|
||||
}
|
||||
|
||||
// fill the list with error texts
|
||||
$('#import-status-error-list').empty();
|
||||
for (var i = 0; i < data.errors.length; i++) {
|
||||
var item = $('<li>').html(data.errors[i]);
|
||||
$('#import-status-error-list').append(item);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// function reportErrors(data) {
|
||||
// "use strict";
|
||||
// if (data.errors.length === 1) {
|
||||
// $('#import-status-error-intro').text(langImportSingleError);
|
||||
// //'An error has occured during the import. The import can continue, however.'
|
||||
// }
|
||||
// if (data.errors.length > 1) {
|
||||
// // 'Errors have occured during the import. The import can continue, however.'
|
||||
// $('#import-status-error-intro').text(langImportMultiError);
|
||||
// }
|
||||
//
|
||||
// // fill the list with error texts
|
||||
// $('#import-status-error-list').empty();
|
||||
// for (var i = 0; i < data.errors.length; i++) {
|
||||
// var item = $('<li>').html(data.errors[i]);
|
||||
// $('#import-status-error-list').append(item);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
function reportStatus(data) {
|
||||
"use strict";
|
||||
console.log('Going to report status: ' + data.statusText);
|
||||
$('#import-status-txt').removeClass('text-danger').text(data.statusText);
|
||||
}
|
||||
|
||||
function kickStartJob() {
|
||||
"use strict";
|
||||
$.post(jobStartUrl);
|
||||
startedTheImport();
|
||||
startedImport = true;
|
||||
}
|
||||
|
||||
function updateTimeout(data) {
|
||||
"use strict";
|
||||
if (data.stepsDone !== stepCount) {
|
||||
stepCount = data.stepsDone;
|
||||
currentLimit = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
currentLimit = currentLimit + interval;
|
||||
}
|
||||
|
||||
function timeoutError() {
|
||||
"use strict";
|
||||
// set status
|
||||
$('#import-status-txt').addClass('text-danger').text(langImportTimeOutError);
|
||||
|
||||
// remove progress bar.
|
||||
$('#import-status-holder').hide();
|
||||
|
||||
}
|
||||
|
||||
function importJobFinished(data) {
|
||||
"use strict";
|
||||
return data.finished;
|
||||
}
|
||||
|
||||
function finishedJob(data) {
|
||||
"use strict";
|
||||
// "There was an error during the import routine. Please check the log files. The error seems to be: '"
|
||||
$('#import-status-txt').removeClass('text-danger').addClass('text-success').text(langImportFinished);
|
||||
|
||||
// remove progress bar.
|
||||
$('#import-status-holder').hide();
|
||||
|
||||
// show info:
|
||||
$('#import-status-intro').show();
|
||||
$('#import-status-more-info').html(data.finishedText).show();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
function startedTheImport() {
|
||||
"use strict";
|
||||
setTimeout(checkImportStatus, interval);
|
||||
}
|
||||
//
|
||||
// function kickStartJob() {
|
||||
// "use strict";
|
||||
// $.post(jobStartUrl);
|
||||
// startedTheImport();
|
||||
// startedImport = true;
|
||||
// }
|
||||
//
|
||||
// function updateTimeout(data) {
|
||||
// "use strict";
|
||||
// if (data.stepsDone !== stepCount) {
|
||||
// stepCount = data.stepsDone;
|
||||
// currentLimit = 0;
|
||||
// return;
|
||||
// }
|
||||
//
|
||||
// currentLimit = currentLimit + interval;
|
||||
// }
|
||||
//
|
||||
// function timeoutError() {
|
||||
// "use strict";
|
||||
// // set status
|
||||
// $('#import-status-txt').addClass('text-danger').text(langImportTimeOutError);
|
||||
//
|
||||
// // remove progress bar.
|
||||
// $('#import-status-holder').hide();
|
||||
//
|
||||
// }
|
||||
//
|
||||
// function importJobFinished(data) {
|
||||
// "use strict";
|
||||
// return data.finished;
|
||||
// }
|
||||
//
|
||||
// function finishedJob(data) {
|
||||
// "use strict";
|
||||
// // "There was an error during the import routine. Please check the log files. The error seems to be: '"
|
||||
// $('#import-status-txt').removeClass('text-danger').addClass('text-success').text(langImportFinished);
|
||||
//
|
||||
// // remove progress bar.
|
||||
// $('#import-status-holder').hide();
|
||||
//
|
||||
// // show info:
|
||||
// $('#import-status-intro').show();
|
||||
// $('#import-status-more-info').html(data.finishedText).show();
|
||||
//
|
||||
// }
|
||||
//
|
||||
//
|
||||
//
|
||||
// function startedTheImport() {
|
||||
// "use strict";
|
||||
// setTimeout(checkImportStatus, interval);
|
||||
// }
|
||||
|
||||
function failedJobImport(jqxhr, textStatus, error) {
|
||||
"use strict";
|
||||
console.error('Job status failed!');
|
||||
// set status
|
||||
// "There was an error during the import routine. Please check the log files. The error seems to be: '"
|
||||
$('#import-status-txt').addClass('text-danger').text(langImportFatalError + ' ' + textStatus + ' ' + error);
|
||||
|
||||
@ -987,7 +987,7 @@ return [
|
||||
'import_file_help' => 'Select your file',
|
||||
'import_status_settings_complete' => 'The import is ready to start.',
|
||||
'import_status_import_complete' => 'The import has completed.',
|
||||
'import_status_import_running' => 'The import is currently running. Please be patient.',
|
||||
'import_status_running' => 'The import is currently running. Please be patient.',
|
||||
'import_status_header' => 'Import status and progress',
|
||||
'import_status_errors' => 'Import errors',
|
||||
'import_status_report' => 'Import report',
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
|
||||
{# Initial display. Will refresh (and disappear almost immediately. #}
|
||||
|
||||
<div class="row status_initial">
|
||||
<div class="row status_initial" style="display:none;">
|
||||
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
|
||||
<div class="box box-primary">
|
||||
<div class="box-header with-border">
|
||||
@ -54,6 +54,43 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row status_running" style="display: none;">
|
||||
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
|
||||
<div class="box box-primary">
|
||||
<div class="box-header with-border">
|
||||
<h3 class="box-title" id="import-status-title">Job status</h3>
|
||||
</div>
|
||||
<div class="box-body">
|
||||
<div id="import-status-holder">
|
||||
<div class="progress" id="import-status-holder">
|
||||
<div id="import-status-bar" class="progress-bar progress-bar-info active progress-bar-striped" role="progressbar"
|
||||
aria-valuenow="100" aria-valuemin="0"
|
||||
aria-valuemax="100" style="width: 100%">
|
||||
</div>
|
||||
</div>
|
||||
<p id="import-status-txt">{{ 'import_status_settings_complete'|_ }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row status_finished" style="display:none;">
|
||||
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
|
||||
<div class="box box-default">
|
||||
<div class="box-header with-border">
|
||||
<h3 class="box-title">{{ 'import_status_report'|_ }}</h3>
|
||||
</div>
|
||||
<div class="box-body">
|
||||
<p id="import-status-intro">
|
||||
{{ 'import_finished_report'|_ }}
|
||||
</p>
|
||||
<p id="import-status-more-info"></p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
{#
|
||||
|
||||
Loading…
Reference in New Issue
Block a user