IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

Improved code for new import + some tests.

Browse Source
This commit is contained in:
James Cole
2018-05-04 20:21:27 +02:00
parent b541f7b944
commit 1c0da454db
22 changed files with 2770 additions and 2281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
app/Http/Controllers/Import/IndexController.php
View File

@@ -24,7 +24,6 @@ namespace FireflyIII\Http\Controllers\Import;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Controllers\Controller;
use FireflyIII\Http\Middleware\IsDemoUser;
use FireflyIII\Import\Prerequisites\PrerequisitesInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use View;
@@ -54,7 +53,6 @@ class IndexController extends Controller
return $next($request);
}
);
$this->middleware(IsDemoUser::class)->except(['index']);
}
/**
@@ -68,11 +66,19 @@ class IndexController extends Controller
*/
public function create(string $importProvider)
{
if (
!(bool)config('app.debug')
&& !(bool)config(sprintf('import.enabled.%s', $importProvider)) === true
&& !\in_array(config('app.env'), ['demo', 'testing'])
) {
throw new FireflyException(sprintf('Import using provider "%s" is currently not available.', $importProvider)); // @codeCoverageIgnore
}
$importJob = $this->repository->create($importProvider);
// if job provider has no prerequisites:
if (!(bool)config(sprintf('import.has_prereq.%s', $importProvider))) {
// @codeCoverageIgnoreStart
// if job provider also has no configuration:
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
$this->repository->updateStatus($importJob, 'ready_to_run');
@@ -85,6 +91,7 @@ class IndexController extends Controller
// redirect to job configuration.
return redirect(route('import.job.configuration.index', [$importJob->key]));
// @codeCoverageIgnoreEnd
}
// if need to set prerequisites, do that first.
@@ -122,7 +129,7 @@ class IndexController extends Controller
$config = config('import.enabled');
$providers = [];
foreach ($config as $name => $enabled) {
if ($enabled || (bool)config('app.debug')) {
if ($enabled || (bool)config('app.debug') || \in_array(config('app.env'), ['demo', 'testing'])) {
$providers[$name] = [];
}
}
@@ -147,110 +154,4 @@ class IndexController extends Controller
return view('import.index', compact('subTitle', 'subTitleIcon', 'providers'));
}
//
// /**
// * @param Request $request
// * @param string $bank
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
// */
// public function reset(Request $request, string $bank)
// {
// if ($bank === 'bunq') {
// // remove bunq related preferences.
// Preferences::delete('bunq_api_key');
// Preferences::delete('bunq_server_public_key');
// Preferences::delete('bunq_private_key');
// Preferences::delete('bunq_public_key');
// Preferences::delete('bunq_installation_token');
// Preferences::delete('bunq_installation_id');
// Preferences::delete('bunq_device_server_id');
// Preferences::delete('external_ip');
//
// }
//
// if ($bank === 'spectre') {
// // remove spectre related preferences:
// Preferences::delete('spectre_client_id');
// Preferences::delete('spectre_app_secret');
// Preferences::delete('spectre_service_secret');
// Preferences::delete('spectre_app_id');
// Preferences::delete('spectre_secret');
// Preferences::delete('spectre_private_key');
// Preferences::delete('spectre_public_key');
// Preferences::delete('spectre_customer');
// }
//
// Preferences::mark();
// $request->session()->flash('info', (string)trans('firefly.settings_reset_for_' . $bank));
//
// return redirect(route('import.index'));
//
// }
// /**
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\JsonResponse
// *
// * @throws FireflyException
// */
// public function start(ImportJob $job)
// {
// $type = $job->file_type;
// $key = sprintf('import.routine.%s', $type);
// $className = config($key);
// if (null === $className || !class_exists($className)) {
// throw new FireflyException(sprintf('Cannot find import routine class for job of type "%s".', $type)); // @codeCoverageIgnore
// }
//
// /** @var RoutineInterface $routine */
// $routine = app($className);
// $routine->setJob($job);
// $result = $routine->run();
//
// if ($result) {
// return response()->json(['run' => 'ok']);
// }
//
// throw new FireflyException('Job did not complete successfully. Please review the log files.');
// }
// /**
// * Generate a JSON file of the job's configuration and send it to the user.
// *
// * @param ImportJob $job
// *
// * @return LaravelResponse
// */
// public function download(ImportJob $job)
// {
// Log::debug('Now in download()', ['job' => $job->key]);
// $config = $job->configuration;
//
// // This is CSV import specific:
// $config['column-roles-complete'] = false;
// $config['column-mapping-complete'] = false;
// $config['initial-config-complete'] = false;
// $config['has-file-upload'] = false;
// $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter'];
// unset($config['stage']);
//
// $result = json_encode($config, JSON_PRETTY_PRINT);
// $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\'));
//
// /** @var LaravelResponse $response */
// $response = response($result, 200);
// $response->header('Content-disposition', 'attachment; filename=' . $name)
// ->header('Content-Type', 'application/json')
// ->header('Content-Description', 'File Transfer')
// ->header('Connection', 'Keep-Alive')
// ->header('Expires', '0')
// ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0')
// ->header('Pragma', 'public')
// ->header('Content-Length', \strlen($result));
//
// return $response;
// }
}

16
app/Http/Controllers/Import/JobConfigurationController.php
View File

@@ -70,10 +70,10 @@ class JobConfigurationController extends Controller
public function index(ImportJob $importJob)
{
// catch impossible status:
$allowed = ['has_prereq', 'need_job_config', 'has_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
$allowed = ['has_prereq', 'need_job_config'];
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
session()->flash('error', trans('import.bad_job_status', ['status' => $importJob->status]));
return redirect(route('import.index'));
}
@@ -82,10 +82,12 @@ class JobConfigurationController extends Controller
// if provider has no config, just push it through:
$importProvider = $importJob->provider;
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
// @codeCoverageIgnoreStart
Log::debug('Job needs no config, is ready to run!');
$this->repository->updateStatus($importJob ,'ready_to_run');
$this->repository->updateStatus($importJob, 'ready_to_run');
return redirect(route('import.job.status.index', [$importJob->key]));
// @codeCoverageIgnoreEnd
}
// create configuration class:
@@ -120,10 +122,10 @@ class JobConfigurationController extends Controller
public function post(Request $request, ImportJob $importJob)
{
// catch impossible status:
$allowed = ['has_prereq', 'need_job_config', 'has_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
$allowed = ['has_prereq', 'need_job_config'];
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
session()->flash('error', trans('import.bad_job_status', ['status' => $importJob->status]));
return redirect(route('import.index'));
}

169
app/Http/Controllers/Import/JobStatusController.php
View File

@@ -32,7 +32,6 @@ use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Http\JsonResponse;
use Log;
use Symfony\Component\Debug\Exception\FatalThrowableError;
/**
* Class JobStatusController
@@ -68,19 +67,6 @@ class JobStatusController extends Controller
*/
public function index(ImportJob $importJob)
{
// jump away depending on job status:
if ($importJob->status === 'has_prereq') {
// TODO back to configuration.
}
if ($importJob->status === 'errored') {
// TODO to error screen
}
if ($importJob->status === 'finished') {
// TODO to finished screen.
}
$subTitleIcon = 'fa-gear';
$subTitle = trans('import.job_status_breadcrumb', ['key' => $importJob->key]);
@@ -94,42 +80,44 @@ class JobStatusController extends Controller
*/
public function json(ImportJob $importJob): JsonResponse
{
$extendedStatus = $importJob->extended_status;
$count = \count($importJob->transactions);
$json = [
$count = \count($importJob->transactions);
$json = [
'status' => $importJob->status,
'errors' => $importJob->errors,
'count' => $count,
'tag_id' => $importJob->tag_id,
'tag_name' => null === $importJob->tag_id ? null : $importJob->tag->tag,
'journals' => $extendedStatus['count'] ?? 0,
'report_txt' => trans('import.unknown_import_result'),
];
// if count is zero:
if (null !== $importJob->tag_id) {
$count = $importJob->tag->transactionJournals->count();
}
if ($count === 0) {
$json['report_txt'] = trans('import.result_no_transactions');
}
if ($count === 1 && null !== $importJob->tag_id) {
$json['report_txt'] = trans('import.result_one_transaction', ['route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag ]);
if ($count === 1) {
$json['report_txt'] = trans('import.result_one_transaction', ['route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag]);
}
if ($count > 1 && null !== $importJob->tag_id) {
$json['report_txt'] = trans('import.result_many_transactions', ['count' => $count,'route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag ]);
$json['report_txt'] = trans(
'import.result_many_transactions', ['count' => $count, 'route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag]
);
}
return response()->json($json);
}
/**
* @param ImportJob $job
* @param ImportJob $importJob
*
* @return JsonResponse
* @throws FireflyException
*/
public function start(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['ready_to_run', 'need_job_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
Log::error('Job is not ready.');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "ready_to_run".']);
@@ -139,29 +127,13 @@ class JobStatusController extends Controller
$key = sprintf('import.routine.%s', $importProvider);
$className = config($key);
if (null === $className || !class_exists($className)) {
return response()->json(['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]);
// @codeCoverageIgnoreStart
return response()->json(
['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]
);
// @codeCoverageIgnoreEnd
}
// if the job is set to "provider_finished", we should be able to store transactions
// generated by the provider.
// otherwise, just continue.
if ($importJob->status === 'provider_finished') {
try {
$this->importFromJob($importJob);
} catch (FireflyException $e) {
$message = 'The import storage routine crashed: ' . $e->getMessage();
Log::error($message);
Log::error($e->getTraceAsString());
// set job errored out:
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => $message]);
}
}
// set job to be running:
$this->repository->setStatus($importJob, 'running');
@@ -170,7 +142,7 @@ class JobStatusController extends Controller
$routine->setJob($importJob);
try {
$routine->run();
} catch (FireflyException $e) {
} catch (FireflyException|Exception $e) {
$message = 'The import routine crashed: ' . $e->getMessage();
Log::error($message);
Log::error($e->getTraceAsString());
@@ -186,16 +158,20 @@ class JobStatusController extends Controller
}
/**
* @param ImportJob $job
* Store does three things:
*
* - Store the transactions.
* - Add them to a tag.
*
* @param ImportJob $importJob
*
* @return JsonResponse
* @throws FireflyException
*/
public function store(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['provider_finished', 'storing_data']; // todo remove storing data.
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
$allowed = ['provider_finished', 'storing_data'];
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
Log::error('Job is not ready.');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "provider_finished".']);
@@ -205,7 +181,7 @@ class JobStatusController extends Controller
$this->repository->setStatus($importJob, 'storing_data');
try {
$this->importFromJob($importJob);
$this->storeTransactions($importJob);
} catch (FireflyException $e) {
$message = 'The import storage routine crashed: ' . $e->getMessage();
Log::error($message);
@@ -216,9 +192,9 @@ class JobStatusController extends Controller
return response()->json(['status' => 'NOK', 'message' => $message]);
}
// set storage to be finished:
$this->repository->setStatus($importJob, 'storage_finished');
// set job to be finished.
$this->repository->setStatus($importJob, 'finished');
// expect nothing from routine, just return OK to user.
return response()->json(['status' => 'OK', 'message' => 'storage_finished']);
@@ -229,90 +205,15 @@ class JobStatusController extends Controller
*
* @throws FireflyException
*/
private function importFromJob(ImportJob $importJob): void
private function storeTransactions(ImportJob $importJob): void
{
/** @var ImportArrayStorage $storage */
$storage = app(ImportArrayStorage::class);
$storage->setJob($importJob);
try {
$storage = new ImportArrayStorage($importJob);
$journals = $storage->store();
$extendedStatus = $importJob->extended_status;
$extendedStatus['count'] = $journals->count();
$this->repository->setExtendedStatus($importJob, $extendedStatus);
} catch (FireflyException|Exception|FatalThrowableError $e) {
$storage->store();
} catch (FireflyException|Exception $e) {
throw new FireflyException($e->getMessage());
}
}
// /**
// * @param ImportJob $job
// *
// * @return \Illuminate\Contracts\View\Factory|\Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|\Illuminate\View\View
// */
// public function index(ImportJob $job)
// {
// $statuses = ['configured', 'running', 'finished', 'error'];
// if (!\in_array($job->status, $statuses)) {
// return redirect(route('import.configure', [$job->key]));
// }
// $subTitle = trans('import.status_sub_title');
// $subTitleIcon = 'fa-star';
//
// return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));
// }
//
// /**
// * Show status of import job in JSON.
// *
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\JsonResponse
// */
// public function json(ImportJob $job)
// {
// $result = [
// 'started' => false,
// 'finished' => false,
// 'running' => false,
// 'errors' => array_values($job->extended_status['errors']),
// 'percentage' => 0,
// 'show_percentage' => false,
// 'steps' => $job->extended_status['steps'],
// 'done' => $job->extended_status['done'],
// 'statusText' => trans('import.status_job_' . $job->status),
// 'status' => $job->status,
// 'finishedText' => '',
// ];
//
// if (0 !== $job->extended_status['steps']) {
// $result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0);
// $result['show_percentage'] = true;
// }
// if ('finished' === $job->status) {
// $result['finished'] = true;
// $tagId = (int)$job->extended_status['tag'];
// if ($tagId !== 0) {
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
// $count = $tag->transactionJournals()->count();
// $result['finishedText'] = trans(
// 'import.status_finished_job', ['count' => $count, 'link' => route('tags.show', [$tag->id, 'all']), 'tag' => $tag->tag]
// );
// }
//
// if ($tagId === 0) {
// $result['finishedText'] = trans('import.status_finished_no_tag'); // @codeCoverageIgnore
// }
// }
//
// if ('running' === $job->status) {
// $result['started'] = true;
// $result['running'] = true;
// }
// $result['percentage'] = $result['percentage'] > 100 ? 100 : $result['percentage'];
// Log::debug(sprintf('JOB STATUS: %d/%d', $result['done'], $result['steps']));
//
// return response()->json($result);
// }
}

2
app/Http/Controllers/Import/PrerequisitesController.php
View File

@@ -51,7 +51,6 @@ class PrerequisitesController extends Controller
function ($request, $next) {
app('view')->share('mainTitleIcon', 'fa-archive');
app('view')->share('title', trans('firefly.import_index_title'));
app('view')->share('subTitleIcon', 'fa-check');
$this->repository = app(ImportJobRepositoryInterface::class);
@@ -59,7 +58,6 @@ class PrerequisitesController extends Controller
return $next($request);
}
);
$this->middleware(IsDemoUser::class);
}
/**

1650
app/Import/Routine/BunqRoutine.php
View File

File diff suppressed because it is too large Load Diff

535
app/Import/Routine/FileRoutine.php
View File

@@ -24,6 +24,7 @@ namespace FireflyIII\Import\Routine;
use Carbon\Carbon;
use DB;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
@@ -38,271 +39,295 @@ use Log;
*/
class FileRoutine implements RoutineInterface
{
/** @var Collection */
public $errors;
/** @var Collection */
public $journals;
/** @var int */
public $lines = 0;
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* ImportRoutine constructor.
*/
public function __construct()
{
$this->journals = new Collection;
$this->errors = new Collection;
}
/**
* @return Collection
*/
public function getErrors(): Collection
{
return $this->errors;
}
/**
* @return Collection
*/
public function getJournals(): Collection
{
return $this->journals;
}
/**
* @return int
*/
public function getLines(): int
{
return $this->lines;
}
// /** @var Collection */
// public $errors;
// /** @var Collection */
// public $journals;
// /** @var int */
// public $lines = 0;
// /** @var ImportJob */
// private $job;
//
// /** @var ImportJobRepositoryInterface */
// private $repository;
//
// /**
// * ImportRoutine constructor.
// */
// public function __construct()
// {
// $this->journals = new Collection;
// $this->errors = new Collection;
// }
//
// /**
// * @return Collection
// */
// public function getErrors(): Collection
// {
// return $this->errors;
// }
//
// /**
// * @return Collection
// */
// public function getJournals(): Collection
// {
// return $this->journals;
// }
//
// /**
// * @return int
// */
// public function getLines(): int
// {
// return $this->lines;
// }
//
// /**
// *
// */
// public function run(): bool
// {
// if ('configured' !== $this->getStatus()) {
// Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
//
// return false;
// }
// set_time_limit(0);
// Log::info(sprintf('Start with import job %s', $this->job->key));
//
// // total steps: 6
// $this->setTotalSteps(6);
//
// $importObjects = $this->getImportObjects();
// $this->lines = $importObjects->count();
// $this->addStep();
//
// // total steps can now be extended. File has been scanned. 7 steps per line:
// $this->addTotalSteps(7 * $this->lines);
//
// // once done, use storage thing to actually store them:
// Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
//
// $storage = $this->storeObjects($importObjects);
// $this->addStep();
// Log::debug('Back in run()');
//
// Log::debug('Updated job...');
// Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
// $this->journals = $storage->journals;
// $this->errors = $storage->errors;
//
// Log::debug('Going to call createImportTag()');
//
// // create tag, link tag to all journals:
// $this->createImportTag();
// $this->addStep();
//
// // update job:
// $this->setStatus('finished');
//
// Log::info(sprintf('Done with import job %s', $this->job->key));
//
// return true;
// }
//
// /**
// * @param ImportJob $job
// */
// public function setJob(ImportJob $job)
// {
// $this->job = $job;
// $this->repository = app(ImportJobRepositoryInterface::class);
// $this->repository->setUser($job->user);
// }
//
// /**
// * @return Collection
// */
// protected function getImportObjects(): Collection
// {
// $objects = new Collection;
// $fileType = $this->getConfig()['file-type'] ?? 'csv';
// // will only respond to "file"
// $class = config(sprintf('import.options.file.processors.%s', $fileType));
// /** @var FileProcessorInterface $processor */
// $processor = app($class);
// $processor->setJob($this->job);
//
// if ('configured' === $this->getStatus()) {
// // set job as "running"...
// $this->setStatus('running');
//
// Log::debug('Job is configured, start with run()');
// $processor->run();
// $objects = $processor->getObjects();
// }
//
// return $objects;
// }
//
// /**
// * Shorthand method.
// */
// private function addStep()
// {
// $this->repository->addStepsDone($this->job, 1);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function addTotalSteps(int $steps)
// {
// $this->repository->addTotalSteps($this->job, $steps);
// }
//
// /**
// *
// */
// private function createImportTag(): Tag
// {
// Log::debug('Now in createImportTag()');
//
// if ($this->journals->count() < 1) {
// Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
//
// return new Tag;
// }
// $this->addTotalSteps($this->journals->count() + 2);
//
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $repository->setUser($this->job->user);
// $data = [
// 'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
// 'date' => new Carbon,
// 'description' => null,
// 'latitude' => null,
// 'longitude' => null,
// 'zoomLevel' => null,
// 'tagMode' => 'nothing',
// ];
// $tag = $repository->store($data);
// $this->addStep();
// $extended = $this->getExtendedStatus();
// $extended['tag'] = $tag->id;
// $this->setExtendedStatus($extended);
//
// Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
// Log::debug('Looping journals...');
// $journalIds = $this->journals->pluck('id')->toArray();
// $tagId = $tag->id;
// foreach ($journalIds as $journalId) {
// Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
// DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
// $this->addStep();
// }
// Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
// $this->addStep();
//
// return $tag;
// }
//
// /**
// * Shorthand method
// *
// * @return array
// */
// private function getConfig(): array
// {
// return $this->repository->getConfiguration($this->job);
// }
//
// /**
// * @return array
// */
// private function getExtendedStatus(): array
// {
// return $this->repository->getExtendedStatus($this->job);
// }
//
// /**
// * Shorthand method.
// *
// * @return string
// */
// private function getStatus(): string
// {
// return $this->repository->getStatus($this->job);
// }
//
// /**
// * @param array $extended
// */
// private function setExtendedStatus(array $extended): void
// {
// $this->repository->setExtendedStatus($this->job, $extended);
// }
//
// /**
// * Shorthand
// *
// * @param string $status
// */
// private function setStatus(string $status): void
// {
// $this->repository->setStatus($this->job, $status);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function setTotalSteps(int $steps)
// {
// $this->repository->setTotalSteps($this->job, $steps);
// }
//
// /**
// * @param Collection $objects
// *
// * @return ImportStorage
// */
// private function storeObjects(Collection $objects): ImportStorage
// {
// $config = $this->getConfig();
// $storage = new ImportStorage;
// $storage->setJob($this->job);
// $storage->setDateFormat($config['date-format']);
// $storage->setObjects($objects);
// $storage->store();
// Log::info('Back in storeObjects()');
//
// return $storage;
// }
/**
* At the end of each run(), the import routine must set the job to the expected status.
*
* The final status of the routine must be "provider_finished".
*
* @return bool
* @throws FireflyException
*/
public function run(): bool
public function run(): void
{
if ('configured' !== $this->getStatus()) {
Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
return false;
}
set_time_limit(0);
Log::info(sprintf('Start with import job %s', $this->job->key));
// total steps: 6
$this->setTotalSteps(6);
$importObjects = $this->getImportObjects();
$this->lines = $importObjects->count();
$this->addStep();
// total steps can now be extended. File has been scanned. 7 steps per line:
$this->addTotalSteps(7 * $this->lines);
// once done, use storage thing to actually store them:
Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
$storage = $this->storeObjects($importObjects);
$this->addStep();
Log::debug('Back in run()');
Log::debug('Updated job...');
Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
$this->journals = $storage->journals;
$this->errors = $storage->errors;
Log::debug('Going to call createImportTag()');
// create tag, link tag to all journals:
$this->createImportTag();
$this->addStep();
// update job:
$this->setStatus('finished');
Log::info(sprintf('Done with import job %s', $this->job->key));
return true;
// TODO: Implement run() method.
throw new NotImplementedException;
}
/**
* @param ImportJob $job
*
* @return mixed
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* @return Collection
*/
protected function getImportObjects(): Collection
{
$objects = new Collection;
$fileType = $this->getConfig()['file-type'] ?? 'csv';
// will only respond to "file"
$class = config(sprintf('import.options.file.processors.%s', $fileType));
/** @var FileProcessorInterface $processor */
$processor = app($class);
$processor->setJob($this->job);
if ('configured' === $this->getStatus()) {
// set job as "running"...
$this->setStatus('running');
Log::debug('Job is configured, start with run()');
$processor->run();
$objects = $processor->getObjects();
}
return $objects;
}
/**
* Shorthand method.
*/
private function addStep()
{
$this->repository->addStepsDone($this->job, 1);
}
/**
* Shorthand
*
* @param int $steps
*/
private function addTotalSteps(int $steps)
{
$this->repository->addTotalSteps($this->job, $steps);
}
/**
*
*/
private function createImportTag(): Tag
{
Log::debug('Now in createImportTag()');
if ($this->journals->count() < 1) {
Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
return new Tag;
}
$this->addTotalSteps($this->journals->count() + 2);
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->job->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
$this->addStep();
$extended = $this->getExtendedStatus();
$extended['tag'] = $tag->id;
$this->setExtendedStatus($extended);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $this->journals->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
$this->addStep();
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
$this->addStep();
return $tag;
}
/**
* Shorthand method
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @return array
*/
private function getExtendedStatus(): array
{
return $this->repository->getExtendedStatus($this->job);
}
/**
* Shorthand method.
*
* @return string
*/
private function getStatus(): string
{
return $this->repository->getStatus($this->job);
}
/**
* @param array $extended
*/
private function setExtendedStatus(array $extended): void
{
$this->repository->setExtendedStatus($this->job, $extended);
}
/**
* Shorthand
*
* @param string $status
*/
private function setStatus(string $status): void
{
$this->repository->setStatus($this->job, $status);
}
/**
* Shorthand
*
* @param int $steps
*/
private function setTotalSteps(int $steps)
{
$this->repository->setTotalSteps($this->job, $steps);
}
/**
* @param Collection $objects
*
* @return ImportStorage
*/
private function storeObjects(Collection $objects): ImportStorage
{
$config = $this->getConfig();
$storage = new ImportStorage;
$storage->setJob($this->job);
$storage->setDateFormat($config['date-format']);
$storage->setObjects($objects);
$storage->store();
Log::info('Back in storeObjects()');
return $storage;
// TODO: Implement setJob() method.
throw new NotImplementedException;
}
}

1070
app/Import/Routine/SpectreRoutine.php
View File

File diff suppressed because it is too large Load Diff

268
app/Import/Storage/ImportArrayStorage.php
View File

@@ -3,21 +3,21 @@
namespace FireflyIII\Import\Storage;
use Carbon\Carbon;
use DB;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Factory\TransactionJournalFactory;
use FireflyIII\Helpers\Collector\JournalCollectorInterface;
use FireflyIII\Helpers\Filter\InternalTransferFilter;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Models\Rule;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use FireflyIII\TransactionRules\Processor;
use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag;
use Log;
use DB;
/**
* Creates new transactions based upon arrays. Will first check the array for duplicates.
@@ -40,11 +40,9 @@ class ImportArrayStorage
private $transfers;
/**
* ImportArrayStorage constructor.
*
* @param ImportJob $importJob
*/
public function __construct(ImportJob $importJob)
public function setJob(ImportJob $importJob): void
{
$this->importJob = $importJob;
$this->countTransfers();
@@ -56,104 +54,61 @@ class ImportArrayStorage
}
/**
* Actually does the storing.
* Actually does the storing. Does three things.
* - Store journals
* - Link to tag
* - Run rules (if set to)
*
* @return Collection
* @throws FireflyException
*/
public function store(): Collection
{
$count = count($this->importJob->transactions);
Log::debug(sprintf('Now in store(). Count of items is %d', $count));
$toStore = [];
foreach ($this->importJob->transactions as $index => $transaction) {
Log::debug(sprintf('Now at item %d out of %d', ($index + 1), $count));
$existingId = $this->hashExists($transaction);
if (null !== $existingId) {
$this->logDuplicateObject($transaction, $existingId);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. It already exists.',
$index, $transaction['description']
)
);
continue;
}
if ($this->checkForTransfers) {
if ($this->transferExists($transaction)) {
$this->logDuplicateTransfer($transaction);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
$index,
$transaction['description']
)
);
continue;
}
}
$toStore[] = $transaction;
// store transactions
$this->setStatus('storing_data');
$collection = $this->storeArray();
$this->setStatus('stored_data');
// link tag:
$this->setStatus('linking_to_tag');
$this->linkToTag($collection);
$this->setStatus('linked_to_tag');
// run rules, if configured to.
$config = $this->importJob->configuration;
if (isset($config['apply-rules']) && $config['apply-rules'] === true) {
$this->setStatus('applying_rules');
$this->applyRules($collection);
$this->setStatus('rules_applied');
}
if (count($toStore) === 0) {
Log::info('No transactions to store left!');
return new Collection;
}
Log::debug('Going to store...');
// now actually store them:
$collection = new Collection;
/** @var TransactionJournalFactory $factory */
$factory = app(TransactionJournalFactory::class);
$factory->setUser($this->importJob->user);
foreach ($toStore as $store) {
// convert the date to an object:
$store['date'] = Carbon::createFromFormat('Y-m-d', $store['date']);
// store the journal.
$collection->push($factory->create($store));
}
Log::debug('DONE storing!');
// create tag and append journals:
$this->createTag($collection);
return $collection;
}
/**
* @param Collection $collection
*
* @throws FireflyException
*/
private function createTag(Collection $collection): void
private function applyRules(Collection $collection): void
{
$rules = $this->getRules();
if ($rules->count() > 0) {
foreach ($collection as $journal) {
$rules->each(
function (Rule $rule) use ($journal) {
Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id));
$processor = Processor::make($rule);
$processor->handleTransactionJournal($journal);
if ($rule->stop_processing) {
return false;
}
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->importJob->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $collection->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
return true;
}
);
}
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $collection->count(), $tag->id, $tag->tag));
$this->repository->setTag($this->importJob, $tag);
}
/**
@@ -161,8 +116,10 @@ class ImportArrayStorage
*/
private function countTransfers(): void
{
/** @var array $array */
$array = $this->importJob->transactions;
$count = 0;
foreach ($this->importJob->transactions as $transaction) {
foreach ($array as $transaction) {
if (strtolower(TransactionType::TRANSFER) === $transaction['type']) {
$count++;
}
@@ -177,6 +134,29 @@ class ImportArrayStorage
}
/**
* @return Collection
*/
private function getRules(): Collection
{
/** @var Collection $set */
$set = Rule::distinct()
->where('rules.user_id', $this->importJob->user_id)
->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id')
->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id')
->where('rule_groups.active', 1)
->where('rule_triggers.trigger_type', 'user_action')
->where('rule_triggers.trigger_value', 'store-journal')
->where('rules.active', 1)
->orderBy('rule_groups.order', 'ASC')
->orderBy('rules.order', 'ASC')
->get(['rules.*', 'rule_groups.order']);
Log::debug(sprintf('Found %d user rules.', $set->count()));
return $set;
}
/**
* Get the users transfers, so they can be compared to whatever the user is trying to import.
*/
@@ -192,7 +172,6 @@ class ImportArrayStorage
}
/**
* @param array $transaction
*
@@ -222,6 +201,39 @@ class ImportArrayStorage
return (int)$entry->transaction_journal_id;
}
/**
* @param Collection $collection
*/
private function linkToTag(Collection $collection): void
{
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->importJob->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $collection->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $collection->count(), $tag->id, $tag->tag));
$this->repository->setTag($this->importJob, $tag);
}
/**
* @param array $transaction
* @param int $existingId
@@ -255,6 +267,84 @@ class ImportArrayStorage
);
}
/**
* Shorthand method to quickly set job status
*
* @param string $status
*/
private function setStatus(string $status): void
{
$this->repository->setStatus($this->importJob, $status);
}
/**
* Store array as journals.
*
* @return Collection
* @throws FireflyException
*/
private function storeArray(): Collection
{
/** @var array $array */
$array = $this->importJob->transactions;
$count = \count($array);
$toStore = [];
Log::debug(sprintf('Now in store(). Count of items is %d', $count));
foreach ($array as $index => $transaction) {
Log::debug(sprintf('Now at item %d out of %d', $index + 1, $count));
$existingId = $this->hashExists($transaction);
if (null !== $existingId) {
$this->logDuplicateObject($transaction, $existingId);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. It already exists.',
$index, $transaction['description']
)
);
continue;
}
if ($this->checkForTransfers) {
if ($this->transferExists($transaction)) {
$this->logDuplicateTransfer($transaction);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
$index,
$transaction['description']
)
);
continue;
}
}
$toStore[] = $transaction;
}
if (\count($toStore) === 0) {
Log::info('No transactions to store left!');
return new Collection;
}
Log::debug('Going to store...');
// now actually store them:
$collection = new Collection;
/** @var TransactionJournalFactory $factory */
$factory = app(TransactionJournalFactory::class);
$factory->setUser($this->importJob->user);
foreach ($toStore as $store) {
// convert the date to an object:
$store['date'] = Carbon::createFromFormat('Y-m-d', $store['date']);
// store the journal.
$collection->push($factory->create($store));
}
Log::debug('DONE storing!');
return $collection;
}
/**
* Check if a transfer exists.
*

2
app/Support/Import/Routine/Fake/StageAhoyHandler.php
View File

@@ -36,7 +36,7 @@ class StageAhoyHandler
*/
public function run(): void
{
for ($i = 0; $i < 15; $i++) {
for ($i = 0; $i < 5; $i++) {
Log::debug(sprintf('Am now in stage AHOY hander, sleeping... (%d)', $i));
sleep(1);
}

2
app/Support/Import/Routine/Fake/StageNewHandler.php
View File

@@ -36,7 +36,7 @@ class StageNewHandler
*/
public function run(): void
{
for ($i = 0; $i < 15; $i++) {
for ($i = 0; $i < 5; $i++) {
Log::debug(sprintf('Am now in stage new hander, sleeping... (%d)', $i));
sleep(1);
}