IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2024-11-29 12:14:34 -06:00
Code Issues Packages Projects Releases Wiki Activity

New code for updated import routine.

Browse Source
This commit is contained in:
James Cole 2018-05-03 17:23:16 +02:00
parent c5142aeba5
commit 6bddb63b45
20 changed files with 843 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Http/Controllers/Import/JobConfigurationController.php
View File

@ -85,7 +85,7 @@ class JobConfigurationController extends Controller
Log::debug('Job needs no config, is ready to run!');
$this->repository->updateStatus($importJob ,'ready_to_run');
return redirect(route('import.job.status.index', [$importProvider->key]));
return redirect(route('import.job.status.index', [$importJob->key]));
}
// create configuration class:

96
app/Http/Controllers/Import/JobStatusController.php
View File

@ -22,14 +22,19 @@ declare(strict_types=1);
namespace FireflyIII\Http\Controllers\Import;
use Exception;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Controllers\Controller;
use FireflyIII\Http\Middleware\IsDemoUser;
use FireflyIII\Import\Routine\RoutineInterface;
use FireflyIII\Import\Storage\ImportArrayStorage;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\RedirectResponse;
use Log;
use Symfony\Component\Debug\Exception\FatalThrowableError;
/**
* Class JobStatusController
@ -82,14 +87,21 @@ class JobStatusController extends Controller
}
/**
* @param ImportJob $job
* @param ImportJob $importJob
*
* @return JsonResponse
*/
public function json(ImportJob $importJob): JsonResponse
{
$extendedStatus = $importJob->extended_status;
$json = [
'status' => $importJob->status,
'errors' => $importJob->errors,
'count' => count($importJob->transactions),
'tag_id' => $importJob->tag_id,
'tag_name' => null === $importJob->tag_id ? null : $importJob->tag->tag,
'journals' => $extendedStatus['count'] ?? 0,
'journals_text' => trans_choice('import.status_with_count', $extendedStatus['count'] ?? 0),
];
return response()->json($json);
@ -104,11 +116,11 @@ class JobStatusController extends Controller
public function start(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['ready_to_run'];
$allowed = ['ready_to_run', 'need_job_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not ready.');
session()->flash('error', trans('import.bad_job_status'));
return redirect(route('import.index'));
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "ready_to_run".']);
}
$importProvider = $importJob->provider;
@ -122,6 +134,21 @@ class JobStatusController extends Controller
// if the job is set to "provider_finished", we should be able to store transactions
// generated by the provider.
// otherwise, just continue.
if ($importJob->status === 'provider_finished') {
try {
$this->importFromJob($importJob);
} catch (FireflyException $e) {
$message = 'The import storage routine crashed: ' . $e->getMessage();
Log::error($message);
Log::error($e->getTraceAsString());
// set job errored out:
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => $message]);
}
}
// set job to be running:
$this->repository->setStatus($importJob, 'running');
@ -144,6 +171,67 @@ class JobStatusController extends Controller
// expect nothing from routine, just return OK to user.
return response()->json(['status' => 'OK', 'message' => 'stage_finished']);
}
/**
* @param ImportJob $job
*
* @return JsonResponse
* @throws FireflyException
*/
public function store(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['provider_finished', 'storing_data']; // todo remove storing data.
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not ready.');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "provider_finished".']);
}
// set job to be storing data:
$this->repository->setStatus($importJob, 'storing_data');
try {
$this->importFromJob($importJob);
} catch (FireflyException $e) {
$message = 'The import storage routine crashed: ' . $e->getMessage();
Log::error($message);
Log::error($e->getTraceAsString());
// set job errored out:
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => $message]);
}
// set job to be finished.
$this->repository->setStatus($importJob, 'finished');
// expect nothing from routine, just return OK to user.
return response()->json(['status' => 'OK', 'message' => 'storage_finished']);
}
/**
* @param ImportJob $importJob
*
* @throws FireflyException
*/
private function importFromJob(ImportJob $importJob): void
{
try {
$storage = new ImportArrayStorage($importJob);
$journals = $storage->store();
$extendedStatus = $importJob->extended_status;
$extendedStatus['count'] = $journals->count();
$this->repository->setExtendedStatus($importJob, $extendedStatus);
} catch (FireflyException|Exception|FatalThrowableError $e) {
throw new FireflyException($e->getMessage());
}
}
// /**

24
app/Import/JobConfiguration/FakeJobConfiguration.php
View File

@ -56,12 +56,18 @@ class FakeJobConfiguration implements JobConfiguratorInterface
// configuration array of job must have two values:
// 'artist' must be 'david bowie', case insensitive
// 'song' must be 'golden years', case insensitive.
// if stage is not "new", then album must be 'station to station'
$config = $this->job->configuration;
if ($this->job->stage === 'new') {
return (isset($config['artist']) && 'david bowie' === strtolower($config['artist']))
&& (isset($config['song']) && 'golden years' === strtolower($config['song']));
}
return isset($config['album']) && 'station to station' === strtolower($config['album']);
}
/**
* Store any data from the $data array into the job.
*
@ -72,16 +78,24 @@ class FakeJobConfiguration implements JobConfiguratorInterface
public function configureJob(array $data): MessageBag
{
$artist = strtolower($data['artist'] ?? '');
$song = strtolower($data['song'] ?? '');
$album = strtolower($data['album'] ?? '');
$configuration = $this->job->configuration;
if ($artist === 'david bowie') {
// store artist
$configuration['artist'] = $artist;
}
$song = strtolower($data['song'] ?? '');
if ($song === 'golden years') {
// store artist
// store song
$configuration['song'] = $song;
}
if ($album=== 'station to station') {
// store album
$configuration['album'] = $album;
}
$this->repository->setConfiguration($this->job, $configuration);
$messages = new MessageBag();
@ -114,12 +128,16 @@ class FakeJobConfiguration implements JobConfiguratorInterface
$config = $this->job->configuration;
$artist = $config['artist'] ?? '';
$song = $config['song'] ?? '';
$album = $config['album'] ?? '';
if (strtolower($artist) !== 'david bowie') {
return 'import.fake.enter-artist';
}
if (strtolower($song) !== 'golden years') {
return 'import.fake.enter-song';
}
if (strtolower($album) !== 'station to station' && $this->job->stage !== 'new') {
return 'import.fake.enter-album';
}
}
/**

2
app/Import/Routine/FakeRoutine.php
View File

@ -87,9 +87,11 @@ class FakeRoutine implements RoutineInterface
break;
case 'final':
$handler = new StageFinalHandler;
$handler->setJob($this->job);
$transactions = $handler->getTransactions();
$this->repository->setStatus($this->job, 'provider_finished');
$this->repository->setStage($this->job, 'final');
$this->repository->setTransactions($this->job, $transactions);
}
}

361
app/Import/Storage/ImportArrayStorage.php Normal file
View File

@ -0,0 +1,361 @@
<?php
namespace FireflyIII\Import\Storage;
use Carbon\Carbon;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Factory\TransactionJournalFactory;
use FireflyIII\Helpers\Collector\JournalCollectorInterface;
use FireflyIII\Helpers\Filter\InternalTransferFilter;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag;
use Log;
use DB;
/**
* Creates new transactions based upon arrays. Will first check the array for duplicates.
*
* Class ImportArrayStorage
*
* @package FireflyIII\Import\Storage
*/
class ImportArrayStorage
{
/** @var bool */
private $checkForTransfers = false;
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var Collection */
private $transfers;
/**
* ImportArrayStorage constructor.
*
* @param ImportJob $importJob
*/
public function __construct(ImportJob $importJob)
{
$this->importJob = $importJob;
$this->countTransfers();
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($importJob->user);
Log::debug('Constructed ImportArrayStorage()');
}
/**
* Actually does the storing.
*
* @return Collection
* @throws FireflyException
*/
public function store(): Collection
{
$count = count($this->importJob->transactions);
Log::debug(sprintf('Now in store(). Count of items is %d', $count));
$toStore = [];
foreach ($this->importJob->transactions as $index => $transaction) {
Log::debug(sprintf('Now at item %d out of %d', ($index + 1), $count));
$existingId = $this->hashExists($transaction);
if (null !== $existingId) {
$this->logDuplicateObject($transaction, $existingId);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. It already exists.',
$index, $transaction['description']
)
);
continue;
}
if ($this->checkForTransfers) {
if ($this->transferExists($transaction)) {
$this->logDuplicateTransfer($transaction);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
$index,
$transaction['description']
)
);
continue;
}
}
$toStore[] = $transaction;
}
if (count($toStore) === 0) {
Log::info('No transactions to store left!');
return new Collection;
}
Log::debug('Going to store...');
// now actually store them:
$collection = new Collection;
/** @var TransactionJournalFactory $factory */
$factory = app(TransactionJournalFactory::class);
$factory->setUser($this->importJob->user);
foreach ($toStore as $store) {
// convert the date to an object:
$store['date'] = Carbon::createFromFormat('Y-m-d', $store['date']);
// store the journal.
$collection->push($factory->create($store));
}
Log::debug('DONE storing!');
// create tag and append journals:
$this->createTag($collection);
return $collection;
}
/**
* @param Collection $collection
*/
private function createTag(Collection $collection): void
{
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->importJob->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $collection->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $collection->count(), $tag->id, $tag->tag));
$this->repository->setTag($this->importJob, $tag);
}
/**
* Count the number of transfers in the array. If this is zero, don't bother checking for double transfers.
*/
private function countTransfers(): void
{
$count = 0;
foreach ($this->importJob->transactions as $transaction) {
if (strtolower(TransactionType::TRANSFER) === $transaction['type']) {
$count++;
}
}
$count = 1;
if ($count > 0) {
$this->checkForTransfers = true;
// get users transfers. Needed for comparison.
$this->getTransfers();
}
}
/**
* Get the users transfers, so they can be compared to whatever the user is trying to import.
*/
private function getTransfers(): void
{
/** @var JournalCollectorInterface $collector */
$collector = app(JournalCollectorInterface::class);
$collector->setAllAssetAccounts()
->setTypes([TransactionType::TRANSFER])
->withOpposingAccount();
$collector->removeFilter(InternalTransferFilter::class);
$this->transfers = $collector->getJournals();
}
/**
* @param array $transaction
*
* @return int|null
* @throws FireflyException
*/
private function hashExists(array $transaction): ?int
{
$json = json_encode($transaction);
if ($json === false) {
throw new FireflyException('Could not encode import array. Please see the logs.', $transaction);
}
$hash = hash('sha256', $json, false);
// find it!
/** @var TransactionJournalMeta $entry */
$entry = TransactionJournalMeta
::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('data', $hash)
->where('name', 'importHashV2')
->first(['journal_meta.*']);
if (null === $entry) {
return null;
}
Log::info(sprintf('Found a transaction journal with an existing hash: %s', $hash));
return (int)$entry->transaction_journal_id;
}
/**
* @param array $transaction
* @param int $existingId
*/
private function logDuplicateObject(array $transaction, int $existingId): void
{
Log::info(
'Transaction is a duplicate, and will not be imported (the hash exists).',
[
'existing' => $existingId,
'description' => $transaction['description'] ?? '',
'amount' => $transaction['transactions'][0]['amount'] ?? 0,
'date' => isset($transaction['date']) ? $transaction['date'] : '',
]
);
}
/**
* @param array $transaction
*/
private function logDuplicateTransfer(array $transaction): void
{
Log::info(
'Transaction is a duplicate transfer, and will not be imported (such a transfer exists already).',
[
'description' => $transaction['description'] ?? '',
'amount' => $transaction['transactions'][0]['amount'] ?? 0,
'date' => isset($transaction['date']) ? $transaction['date'] : '',
]
);
}
/**
* Check if a transfer exists.
*
* @param $transaction
*
* @return bool
*/
private function transferExists(array $transaction): bool
{
Log::debug('Check if is a double transfer.');
if (strtolower(TransactionType::TRANSFER) !== $transaction['type']) {
Log::debug(sprintf('Is a %s, not a transfer so no.', $transaction['type']));
return false;
}
// how many hits do we need?
$requiredHits = count($transaction['transactions']) * 4;
$totalHits = 0;
Log::debug(sprintf('Required hits for transfer comparison is %d', $requiredHits));
// loop over each split:
foreach ($transaction['transactions'] as $current) {
// get the amount:
$amount = (string)($current['amount'] ?? '0');
if (bccomp($amount, '0') === -1) {
$amount = bcmul($amount, '-1');
}
// get the description:
$description = strlen((string)$current['description']) === 0 ? $transaction['description'] : $current['description'];
// get the source and destination ID's:
$currentSourceIDs = [(int)$current['source_id'], (int)$current['destination_id']];
sort($currentSourceIDs);
// get the source and destination names:
$currentSourceNames = [(string)$current['source_name'], (string)$current['destination_name']];
sort($currentSourceNames);
// then loop all transfers:
/** @var Transaction $transfer */
foreach ($this->transfers as $transfer) {
// number of hits for this split-transfer combination:
$hits = 0;
Log::debug(sprintf('Now looking at transaction journal #%d', $transfer->journal_id));
// compare amount:
Log::debug(sprintf('Amount %s compared to %s', $amount, $transfer->transaction_amount));
if (0 !== bccomp($amount, $transfer->transaction_amount)) {
continue;
}
++$hits;
Log::debug(sprintf('Comparison is a hit! (%s)', $hits));
// compare description:
Log::debug(sprintf('Comparing "%s" to "%s"', $description, $transfer->description));
if ($description !== $transfer->description) {
continue;
}
++$hits;
Log::debug(sprintf('Comparison is a hit! (%s)', $hits));
// compare date:
$transferDate = $transfer->date->format('Y-m-d');
Log::debug(sprintf('Comparing dates "%s" to "%s"', $transaction['date'], $transferDate));
if ($transaction['date'] !== $transferDate) {
continue;
}
++$hits;
Log::debug(sprintf('Comparison is a hit! (%s)', $hits));
// compare source and destination id's
$transferSourceIDs = [(int)$transfer->account_id, (int)$transfer->opposing_account_id];
sort($transferSourceIDs);
Log::debug('Comparing current transaction source+dest IDs', $currentSourceIDs);
Log::debug('.. with current transfer source+dest IDs', $transferSourceIDs);
if ($currentSourceIDs === $transferSourceIDs) {
++$hits;
Log::debug(sprintf('Source IDs are the same! (%d)', $hits));
}
unset($transferSourceIDs);
// compare source and destination names
$transferSource = [(string)$transfer->account_name, (int)$transfer->opposing_account_name];
sort($transferSource);
Log::debug('Comparing current transaction source+dest names', $currentSourceNames);
Log::debug('.. with current transfer source+dest names', $transferSource);
if ($currentSourceNames === $transferSource) {
Log::debug(sprintf('Source names are the same! (%d)', $hits));
++$hits;
}
$totalHits += $hits;
if ($totalHits >= $requiredHits) {
return true;
}
}
}
Log::debug(sprintf('Total hits: %d, required: %d', $totalHits, $requiredHits));
return $totalHits >= $requiredHits;
}
}

12
app/Models/ImportJob.php
View File

@ -47,9 +47,10 @@ class ImportJob extends Model
'configuration' => 'array',
'extended_status' => 'array',
'transactions' => 'array',
'errors' => 'array',
];
/** @var array */
protected $fillable = ['key', 'user_id', 'file_type', 'provider', 'status', 'stage', 'configuration', 'extended_status', 'transactions'];
protected $fillable = ['key', 'user_id', 'file_type', 'provider', 'status', 'stage', 'configuration', 'extended_status', 'transactions', 'errors'];
/**
* @param $value
@ -79,4 +80,13 @@ class ImportJob extends Model
{
return $this->belongsTo(User::class);
}
/**
* @codeCoverageIgnore
* @return \Illuminate\Database\Eloquent\Relations\BelongsTo
*/
public function tag()
{
return $this->belongsTo(Tag::class);
}
}

49
app/Repositories/ImportJob/ImportJobRepository.php
View File

@ -25,6 +25,7 @@ namespace FireflyIII\Repositories\ImportJob;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use FireflyIII\User;
@ -126,6 +127,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$importJob = ImportJob::create(
[
'user_id' => $this->user->id,
'tag_id' => null,
'provider' => $importProvider,
'file_type' => '',
'key' => Str::random(12),
@ -134,6 +136,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
'configuration' => [],
'extended_status' => [],
'transactions' => [],
'errors' => [],
]
);
@ -427,4 +430,50 @@ class ImportJobRepository implements ImportJobRepositoryInterface
{
return $job->uploadFileContents();
}
/**
* @param ImportJob $job
* @param array $transactions
*
* @return ImportJob
*/
public function setTransactions(ImportJob $job, array $transactions): ImportJob
{
$job->transactions = $transactions;
$job->save();
return $job;
}
/**
* Add message to job.
*
* @param ImportJob $job
* @param string $error
*
* @return ImportJob
*/
public function addErrorMessage(ImportJob $job, string $error): ImportJob
{
$errors = $job->errors;
$errors[] = $error;
$job->errors = $errors;
$job->save();
return $job;
}
/**
* @param ImportJob $job
* @param Tag $tag
*
* @return ImportJob
*/
public function setTag(ImportJob $job, Tag $tag): ImportJob
{
$job->tag()->associate($tag);
$job->save();
return $job;
}
}

27
app/Repositories/ImportJob/ImportJobRepositoryInterface.php
View File

@ -23,6 +23,7 @@ declare(strict_types=1);
namespace FireflyIII\Repositories\ImportJob;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\User;
use Symfony\Component\HttpFoundation\File\UploadedFile;
@ -32,6 +33,32 @@ use Symfony\Component\HttpFoundation\File\UploadedFile;
interface ImportJobRepositoryInterface
{
/**
* @param ImportJob $job
* @param array $transactions
*
* @return ImportJob
*/
public function setTransactions(ImportJob $job, array $transactions): ImportJob;
/**
* @param ImportJob $job
* @param Tag $tag
*
* @return ImportJob
*/
public function setTag(ImportJob $job, Tag $tag): ImportJob;
/**
* Add message to job.
*
* @param ImportJob $job
* @param string $error
*
* @return ImportJob
*/
public function addErrorMessage(ImportJob $job, string $error): ImportJob;
/**
* @param ImportJob $job
* @param int $index

11
app/Repositories/Tag/TagRepository.php
View File

@ -105,6 +105,7 @@ class TagRepository implements TagRepositoryInterface
/**
* @param int $tagId
*
* @deprecated
* @return Tag
*/
public function find(int $tagId): Tag
@ -453,4 +454,14 @@ class TagRepository implements TagRepositoryInterface
return (int)($range[0] + $extra);
}
/**
* @param int $tagId
*
* @return Tag|null
*/
public function findNull(int $tagId): ?Tag
{
return $this->user->tags()->find($tagId);
}
}

7
app/Repositories/Tag/TagRepositoryInterface.php
View File

@ -69,6 +69,13 @@ interface TagRepositoryInterface
/**
* @param int $tagId
*
* @return Tag|null
*/
public function findNull(int $tagId): ?Tag;
/**
* @param int $tagId
* @deprecated
* @return Tag
*/
public function find(int $tagId): Tag;

2
app/Support/Import/Routine/Fake/StageAhoyHandler.php
View File

@ -36,7 +36,7 @@ class StageAhoyHandler
*/
public function run(): void
{
for ($i = 0; $i < 15; $i++) {
for ($i = 0; $i < 2; $i++) {
Log::debug(sprintf('Am now in stage AHOY hander, sleeping... (%d)', $i));
sleep(1);
}

96
app/Support/Import/Routine/Fake/StageFinalHandler.php
View File

@ -2,6 +2,8 @@
namespace FireflyIII\Support\Import\Routine\Fake;
use Carbon\Carbon;
/**
* Class StageFinalHandler
*
@ -9,6 +11,18 @@ namespace FireflyIII\Support\Import\Routine\Fake;
*/
class StageFinalHandler
{
private $job;
/**
* @param mixed $job
*/
public function setJob($job): void
{
$this->job = $job;
}
/**
* @return array
*/
@ -17,12 +31,92 @@ class StageFinalHandler
$transactions = [];
for ($i = 0; $i < 5; $i++) {
$transaction = [];
$transaction = [
'type' => 'withdrawal',
'date' => Carbon::create()->format('Y-m-d'),
'tags' => '',
'user' => $this->job->user_id,
// all custom fields:
'internal_reference' => null,
'notes' => null,
// journal data:
'description' => 'Some random description #' . random_int(1, 10000),
'piggy_bank_id' => null,
'piggy_bank_name' => null,
'bill_id' => null,
'bill_name' => null,
// transaction data:
'transactions' => [
[
'currency_id' => null,
'currency_code' => 'EUR',
'description' => null,
'amount' => random_int(500, 5000) / 100,
'budget_id' => null,
'budget_name' => null,
'category_id' => null,
'category_name' => null,
'source_id' => null,
'source_name' => 'Checking Account',
'destination_id' => null,
'destination_name' => 'Random expense account #' . random_int(1, 10000),
'foreign_currency_id' => null,
'foreign_currency_code' => null,
'foreign_amount' => null,
'reconciled' => false,
'identifier' => 0,
],
],
];
$transactions[] = $transaction;
}
// add a transfer I know exists already
$transactions[] = [
'type' => 'transfer',
'date' => '2017-02-28',
'tags' => '',
'user' => $this->job->user_id,
// all custom fields:
'internal_reference' => null,
'notes' => null,
// journal data:
'description' => 'Saving money for February',
'piggy_bank_id' => null,
'piggy_bank_name' => null,
'bill_id' => null,
'bill_name' => null,
// transaction data:
'transactions' => [
[
'currency_id' => null,
'currency_code' => 'EUR',
'description' => null,
'amount' => '140',
'budget_id' => null,
'budget_name' => null,
'category_id' => null,
'category_name' => null,
'source_id' => 1,
'source_name' => 'Checking Account',
'destination_id' => 2,
'destination_name' => null,
'foreign_currency_id' => null,
'foreign_currency_code' => null,
'foreign_amount' => null,
'reconciled' => false,
'identifier' => 0,
],
],
];
return $transactions;

2
app/Support/Import/Routine/Fake/StageNewHandler.php
View File

@ -36,7 +36,7 @@ class StageNewHandler
*/
public function run(): void
{
for ($i = 0; $i < 15; $i++) {
for ($i = 0; $i < 2; $i++) {
Log::debug(sprintf('Am now in stage new hander, sleeping... (%d)', $i));
sleep(1);
}

2
config/import.php
View File

@ -65,7 +65,7 @@ return [
'yodlee' => false,
],
'has_config' => [
'fake' => true,
'fake' => false,
'file' => true,
'bunq' => true,
'spectre' => true,

4
database/migrations/2018_04_29_174524_changes_for_v474.php
View File

@ -31,6 +31,10 @@ class ChangesForV474 extends Migration
$table->string('provider', 50)->after('file_type')->default('');
$table->string('stage', 50)->after('status')->default('');
$table->longText('transactions')->after('extended_status');
$table->longText('errors')->after('transactions');
$table->integer('tag_id', false, true)->nullable()->after('user_id');
$table->foreign('tag_id')->references('id')->on('tags')->onDelete('set null');
}
);
}

64
public/js/ff/import/status_v2.js
View File

@ -22,6 +22,7 @@
var timeOutId;
var hasStartedJob = false;
var jobStorageStarted = false;
var checkInitialInterval = 1000;
var checkNextInterval = 500;
var maxLoops = 60;
@ -59,7 +60,8 @@ function reportOnJobStatus(data) {
checkOnJob();
break;
case "running":
showProgressBox();
case "storing_data":
showProgressBox(data.ttatus);
checkOnJob();
break;
@ -67,12 +69,48 @@ function reportOnJobStatus(data) {
// redirect user to configuration for this job.
window.location.replace(jobConfigurationUri);
break;
case 'provider_finished':
// call routine to store stuff:
storeJobData();
checkOnJob();
break;
case "finished":
showJobResults(data);
break;
default:
console.error('Cannot handle status ' + data.status);
}
}
/**
*
* @param data
*/
function showJobResults(data) {
// hide all boxes.
// hide status boxes:
$('.statusbox').hide();
// render the count:
$('#import-status-more-info').append($('<span>').text(data.journals_text));
// render relevant data from JSON thing.
if (data.errors.length > 0) {
$('#import-status-error-txt').show();
data.errors.forEach(function (element) {
$('#import-status-errors').append($('<li>').text(element));
});
}
// show success box.
$('.status_finished').show();
}
/**
* Will refresh and get job status.
*/
@ -100,6 +138,20 @@ function startJob() {
$.post(jobStartUri, {_token: token}).fail(reportOnSubmitError).done(reportOnSubmit)
}
/**
* Start the storage routine for this job.
*/
function storeJobData() {
console.log('In storeJobData()');
if (jobStorageStarted) {
console.log('Store job already started!');
return;
}
console.log('STORAGE JOB STARTED!');
jobStorageStarted = true;
$.post(jobStorageStartUri, {_token: token}).fail(reportOnSubmitError).done(reportOnSubmit)
}
/**
* Function is called when the JSON array could not be retrieved.
*
@ -121,12 +173,20 @@ function reportFailure(xhr, status, error) {
// show error box.
}
function showProgressBox() {
/**
*
*/
function showProgressBox(status) {
// hide fatal error box:
$('.fatal_error').hide();
// hide initial status box:
$('.status_initial').hide();
if(status === 'running') {
$('#import-status-txt').text(langImportRunning);
} else {
$('#import-status-txt').text(langImportStoring);
}
// show running box:
$('.status_running').show();

4
resources/lang/en_US/import.php
View File

@ -38,6 +38,7 @@ return [
'status_job_configuring' => 'The import is being configured.',
'status_job_configured' => 'The import is configured.',
'status_job_running' => 'The import is running.. Please wait..',
'status_job_storing' => 'The import is storing your data.. Please wait..',
'status_job_error' => 'The job has generated an error.',
'status_job_finished' => 'The import has finished!',
'status_running_title' => 'The import is running',
@ -47,12 +48,15 @@ return [
'status_errors_title' => 'Errors during the import',
'status_errors_single' => 'An error has occurred during the import. It does not appear to be fatal.',
'status_errors_multi' => 'Some errors occurred during the import. These do not appear to be fatal.',
'status_with_count' => 'One transaction has been imported|:count transactions have been imported.',
'status_bread_crumb' => 'Import status',
'status_sub_title' => 'Import status',
'config_sub_title' => 'Set up your import',
'status_finished_job' => 'The :count transactions imported can be found in tag <a href=":link" class="label label-success" style="font-size:100%;font-weight:normal;">:tag</a>.',
'status_finished_no_tag' => 'Firefly III has not collected any transactions from your import file.',
'import_with_key' => 'Import with key \':key\'',
'finished_with_errors' => 'The import reported some problems.',
// file, upload something
'file_upload_title' => 'Import setup (1/4) - Upload your file',

53
resources/views/import/fake/enter-album.twig Normal file
View File

@ -0,0 +1,53 @@
{% extends "./layout/default" %}
{% block breadcrumbs %}
{{ Breadcrumbs.render }}
{% endblock %}
{% block content %}
<div class="row">
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">Enter station for fake import</h3>
</div>
<div class="box-body">
<p>
Enter "station to station", no matter the capitalization.
</p>
</div>
</div>
</div>
</div>
<form method="POST" action="{{ route('import.job.configuration.post', importJob.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
<input name="_token" type="hidden" value="{{ csrf_token() }}">
<div class="row">
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">Fields be here.</h3>
</div>
<div class="box-body">
{{ ExpandedForm.text('album') }}
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-lg-12">
<div class="box">
<div class="box-body">
<button type="submit" class="btn btn-success pull-right">
Submit it!
</button>
</div>
</div>
</div>
</div>
</form>
{% endblock %}
{% block scripts %}
{% endblock %}
{% block styles %}
{% endblock %}

23
resources/views/import/status.twig
View File

@ -56,7 +56,7 @@
<span class="sr-only">Running...</span>
</div>
</div>
<p id="import-status-txt">Some text here</p>
<p id="import-status-txt"></p>
</div>
</div>
</div>
@ -128,7 +128,7 @@
</div>
</div>
#}
{# displays the finished status of the import
{# displays the finished status of the import #}
<div class="row status_finished statusbox" style="display:none;">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-default">
@ -138,14 +138,16 @@
<div class="box-body">
<p id="import-status-intro">
{{ trans('import.status_finished_text') }}
<span id="import-status-more-info"></span>
</p>
<p id="import-status-more-info"></p>
</div>
</div>
</div>
</div>
#}
<p id="import-status-error-txt" style="display:none;">{{ trans('import.finished_with_errors') }}</p>
<ul id="import-status-errors" class="text-danger">
</ul>
</div>
</div>
</div>
</div>
{# box to show error information. #}
{#
<div class="row info_errors" style="display:none;">
@ -171,6 +173,11 @@
var jobStatusUri = '{{ route('import.job.status.json', [importJob.key]) }}';
var jobStartUri = '{{ route('import.job.start', [importJob.key]) }}';
var jobConfigurationUri = '{{ route('import.job.configuration.index', [importJob.key]) }}';
var jobStorageStartUri = '{{ route('import.job.store', [importJob.key]) }}';
// import is running:
var langImportRunning = '{{ trans('import.status_job_running') }}';
var langImportStoring = '{{ trans('import.status_job_storing') }}';
// some useful translations.
{#var langImportTimeOutError = '(time out thing)';#}

1
routes/web.php
View File

@ -462,6 +462,7 @@ Route::group(
// start the job!
Route::any('job/start/{importJob}', ['uses' => 'Import\JobStatusController@start', 'as' => 'job.start']);
Route::any('job/store/{importJob}', ['uses' => 'Import\JobStatusController@store', 'as' => 'job.store']);
// import method prerequisites:
#