diff --git a/app/Import/ImportStorage.php b/app/Import/ImportStorage.php index db06eb5251..445dc18caa 100644 --- a/app/Import/ImportStorage.php +++ b/app/Import/ImportStorage.php @@ -15,6 +15,7 @@ use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Transaction; use FireflyIII\Models\TransactionJournal; +use FireflyIII\Models\TransactionJournalMeta; use FireflyIII\Models\TransactionType; use FireflyIII\User; use Illuminate\Support\Collection; @@ -81,6 +82,19 @@ class ImportStorage return $collection; } + /** + * @param string $hash + * + * @return bool + */ + private function alreadyImported(string $hash): bool + { + + $count = TransactionJournalMeta::where('name', 'originalImportHash')->where('data', json_encode($hash))->count(); + + return $count > 0; + } + /** * @param float $amount * @@ -183,6 +197,7 @@ class ImportStorage */ private function storeJournal($entry): TransactionJournal { + $billId = is_null($entry->fields['bill']) ? null : $entry->fields['bill']->id; $journalData = [ 'user_id' => $entry->user->id, @@ -204,6 +219,13 @@ class ImportStorage } Log::debug('Created journal', ['id' => $journal->id]); + // save hash as meta value: + $meta = new TransactionJournalMeta; + $meta->name = 'originalImportHash'; + $meta->data = $entry->hash; + $meta->transactionjournal()->associate($journal); + $meta->save(); + return $journal; } @@ -230,6 +252,21 @@ class ImportStorage return $result; } + + if ($this->alreadyImported($entry->hash)) { + Log::warning(sprintf('Cannot import row %d, because it has already been imported.', $index)); + $result = new ImportResult(); + $result->failed(); + $errorText = sprintf('Row #%d: This row has been imported before.', $index); + $result->appendError($errorText); + $extendedStatus = $this->job->extended_status; + $extendedStatus['errors'][] = $errorText; + $this->job->extended_status = $extendedStatus; + $this->job->save(); + + return $result; + } + Log::debug(sprintf('Going to store row %d', $index)); diff --git a/app/Import/ImportValidator.php b/app/Import/ImportValidator.php index 35e0d08883..e22c2418cb 100644 --- a/app/Import/ImportValidator.php +++ b/app/Import/ImportValidator.php @@ -86,7 +86,6 @@ class ImportValidator $newCollection->put($index, $entry); $this->job->addStepsDone(1); - sleep(1); } Log::notice(sprintf('Finished validating %d entry(ies).', $newCollection->count())); diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/CsvImporter.php index 970529caa0..0594108878 100644 --- a/app/Import/Importer/CsvImporter.php +++ b/app/Import/Importer/CsvImporter.php @@ -66,7 +66,6 @@ class CsvImporter implements ImporterInterface $this->collection->put($line, $importEntry); $this->job->addTotalSteps(3); $this->job->addStepsDone(1); - sleep(1); } } Log::debug(sprintf('Import collection contains %d entries', $this->collection->count())); diff --git a/app/Models/TransactionJournalMeta.php b/app/Models/TransactionJournalMeta.php index 801ab0e3df..a3da2f15c1 100644 --- a/app/Models/TransactionJournalMeta.php +++ b/app/Models/TransactionJournalMeta.php @@ -66,7 +66,7 @@ class TransactionJournalMeta extends Model * * @return \Illuminate\Database\Eloquent\Relations\BelongsTo */ - public function transactionjournal(): BelongsTo + public function transactionJournal(): BelongsTo { return $this->belongsTo('FireflyIII\Models\TransactionJournal'); } diff --git a/public/js/ff/import/status.js b/public/js/ff/import/status.js index 5cb6abb39e..8d98731f1b 100644 --- a/public/js/ff/import/status.js +++ b/public/js/ff/import/status.js @@ -5,12 +5,15 @@ * This software may be modified and distributed under the terms * of the MIT license. See the LICENSE file for details. */ -/* globals $, jobImportUrl, jobStartUrl, token, langImportMultiError, langImportSingleError */ +/* globals $, jobImportUrl, jobStartUrl, token, langImportMultiError, langImportSingleError, langImportFatalError, langImportTimeOutError */ var startedImport = false; var startInterval = 2000; var interval = 500; +var timeoutLimit = 5000; +var currentLimit = 0; +var stepCount = 0; $(function () { "use strict"; @@ -85,12 +88,42 @@ function kickStartJob() { startedImport = true; } +function updateTimeout(data) { + "use strict"; + if (data.stepsDone != stepCount) { + stepCount = data.stepsDone; + currentLimit = 0; + return; + } + + currentLimit = currentLimit + interval; + console.log("stepCount: " + stepCount + ", stepsDone: " + data.stepsDone + ", currentLimit: " + currentLimit); +} + +function timeoutError() { + "use strict"; + + // set status + $('#import-status-txt').addClass('text-danger').text(langImportTimeOutError); + + // remove progress bar. + $('#import-status-holder').hide(); + +} + function reportOnJobImport(data) { "use strict"; updateBar(data); reportErrors(data); reportStatus(data); + updateTimeout(data); + + // same number of steps as last time? + if (currentLimit > timeoutLimit) { + timeoutError(); + return; + } // if the job has not actually started, do so now: if (!data.started && !startedImport) { @@ -113,9 +146,7 @@ function failedJobImport(jqxhr, textStatus, error) { // set status // "There was an error during the import routine. Please check the log files. The error seems to be: '" - $('#import-status-txt').addClass('text-danger').text( - langImportFatalError + textStatus + ' ' + error - ); + $('#import-status-txt').addClass('text-danger').text(langImportFatalError + ' ' + textStatus + ' ' + error); // remove progress bar. $('#import-status-holder').hide(); diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index b952928331..93769ecd60 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -786,4 +786,8 @@ return [ 'import_status_errors' => 'Import errors', 'import_status_report' => 'Import report', 'import_finished' => 'Import has finished', + 'import_error_single' => 'An error has occured during the import.', + 'import_error_multi' => 'Some errors occured during the import.', + 'import_error_fatal' => 'There was an error during the import routine. Please check the log files. The error seems to be:', + 'import_error_timeout' => 'The import seems to have timed out. If this error persists, please import your data using the console command.', ]; diff --git a/resources/views/import/status.twig b/resources/views/import/status.twig index 1ccf7acd4e..9044770356 100644 --- a/resources/views/import/status.twig +++ b/resources/views/import/status.twig @@ -62,6 +62,7 @@ var langImportSingleError = '{{ 'import_error_single'|_ }}'; var langImportMultiError = '{{ 'import_error_multi'|_ }}'; var langImportFatalError = '{{ 'import_error_fatal'|_ }}'; + var langImportTimeOutError = '{{ 'import_error_timeout'|_ }}'; var jobKey = '{{ job.key }}'; var jobImportUrl = '{{ route('import.json', [job.key]) }}';