From 2edd49a8b45a6aad896e750c32571406e869f4af Mon Sep 17 00:00:00 2001 From: James Cole Date: Mon, 8 Jan 2018 20:20:45 +0100 Subject: [PATCH] First version that supports Spectre. --- app/Import/Object/ImportJournal.php | 2 + app/Import/Routine/SpectreRoutine.php | 168 ++++++++++++++++-- app/Import/Storage/ImportStorage.php | 4 +- app/Models/ImportJob.php | 26 +++ .../ImportJob/ImportJobRepository.php | 26 +++ .../ImportJobRepositoryInterface.php | 17 ++ app/Services/Spectre/Object/Account.php | 8 + app/Services/Spectre/Object/Transaction.php | 41 +++++ .../Spectre/Object/TransactionExtra.php | 1 + 9 files changed, 279 insertions(+), 14 deletions(-) diff --git a/app/Import/Object/ImportJournal.php b/app/Import/Object/ImportJournal.php index 1b3ac44a42..34fd59eded 100644 --- a/app/Import/Object/ImportJournal.php +++ b/app/Import/Object/ImportJournal.php @@ -179,6 +179,8 @@ class ImportJournal */ public function setValue(array $array) { + $array['mapped'] = $array['mapped'] ?? null; + $array['value'] = $array['value'] ?? null; switch ($array['role']) { default: throw new FireflyException(sprintf('ImportJournal cannot handle "%s" with value "%s".', $array['role'], $array['value'])); diff --git a/app/Import/Routine/SpectreRoutine.php b/app/Import/Routine/SpectreRoutine.php index 993983e062..fe41577046 100644 --- a/app/Import/Routine/SpectreRoutine.php +++ b/app/Import/Routine/SpectreRoutine.php @@ -22,15 +22,21 @@ declare(strict_types=1); namespace FireflyIII\Import\Routine; +use Carbon\Carbon; +use DB; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Import\Object\ImportJournal; +use FireflyIII\Import\Storage\ImportStorage; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use FireflyIII\Repositories\Tag\TagRepositoryInterface; use FireflyIII\Services\Spectre\Exception\DuplicatedCustomerException; use FireflyIII\Services\Spectre\Exception\SpectreException; use FireflyIII\Services\Spectre\Object\Account; use FireflyIII\Services\Spectre\Object\Customer; use FireflyIII\Services\Spectre\Object\Login; use FireflyIII\Services\Spectre\Object\Token; +use FireflyIII\Services\Spectre\Object\Transaction; use FireflyIII\Services\Spectre\Request\CreateTokenRequest; use FireflyIII\Services\Spectre\Request\ListAccountsRequest; use FireflyIII\Services\Spectre\Request\ListCustomersRequest; @@ -144,10 +150,7 @@ class SpectreRoutine implements RoutineInterface throw new FireflyException(sprintf('Cannot handle stage %s', $stage)); } - var_dump($config); - exit; - - throw new FireflyException('Application cannot handle this.'); + return true; } /** @@ -253,11 +256,17 @@ class SpectreRoutine implements RoutineInterface $customer = $this->getCustomer(); Log::debug(sprintf('Customer ID is %s', $customer->getId())); + // add some steps done + $this->repository->addStepsDone($this->job, 2); + // use customer to request a token: $uri = route('import.status', [$this->job->key]); $token = $this->getToken($customer, $uri); Log::debug(sprintf('Token is %s', $token->getToken())); + // add some steps done + $this->repository->addStepsDone($this->job, 2); + // update job, give it the token: $config = $this->job->configuration; $config['has-token'] = true; @@ -287,6 +296,10 @@ class SpectreRoutine implements RoutineInterface $request = new ListLoginsRequest($this->job->user); $request->setCustomer($customer); $request->call(); + + // add some steps done + $this->repository->addStepsDone($this->job, 2); + $logins = $request->getLogins(); /** @var Login $final */ $final = null; @@ -305,12 +318,18 @@ class SpectreRoutine implements RoutineInterface throw new FireflyException('No valid login attempt found.'); } + // add some steps done + $this->repository->addStepsDone($this->job, 2); + // list the users accounts using this login. $accountRequest = new ListAccountsRequest($this->job->user); $accountRequest->setLogin($login); $accountRequest->call(); $accounts = $accountRequest->getAccounts(); + // add some steps done + $this->repository->addStepsDone($this->job, 2); + // store accounts in job: $all = []; /** @var Account $account */ @@ -327,37 +346,160 @@ class SpectreRoutine implements RoutineInterface $this->job->status = 'configuring'; $this->job->save(); + // add some steps done + $this->repository->addStepsDone($this->job, 2); + return; } /** + * @param array $all * + * @throws FireflyException + */ + private function importTransactions(array $all) + { + Log::debug('Going to import transactions'); + $collection = new Collection; + // create import objects? + foreach ($all as $accountId => $data) { + Log::debug(sprintf('Now at account #%d', $accountId)); + /** @var Transaction $transaction */ + foreach ($data['transactions'] as $transaction) { + Log::debug(sprintf('Now at transaction #%d', $transaction->getId())); + /** @var Account $account */ + $account = $data['account']; + $importJournal = new ImportJournal; + $importJournal->setUser($this->job->user); + $importJournal->asset->setDefaultAccountId($data['import_id']); + // call set value a bunch of times for various data entries: + $tags = []; + $tags[] = $transaction->getMode(); + $tags[] = $transaction->getStatus(); + if ($transaction->isDuplicated()) { + $tags[] = 'possibly-duplicated'; + } + $extra = $transaction->getExtra()->toArray(); + $notes = ''; + $notes .= strval(trans('import.imported_from_account', ['account' => $account->getName()])) . ' ' + . "\n"; // double space for newline in Markdown. + foreach ($extra as $key => $value) { + switch ($key) { + default: + $notes .= $key . ': ' . $value . ' '; // for newline in Markdown. + } + } + // hash + $importJournal->setHash($transaction->getHash()); + + // account ID (Firefly III account): + $importJournal->setValue(['role' => 'account-id', 'value' => $data['import_id'], 'mapped' => $data['import_id']]); + + // description: + $importJournal->setValue(['role' => 'description', 'value' => $transaction->getDescription()]); + + // date: + $importJournal->setValue(['role' => 'date-transaction', 'value' => $transaction->getMadeOn()->toIso8601String()]); + + + // amount + $importJournal->setValue(['role' => 'amount', 'value' => $transaction->getAmount()]); + $importJournal->setValue(['role' => 'currency-code', 'value' => $transaction->getCurrencyCode()]); + + + // various meta fields: + $importJournal->setValue(['role' => 'category-name', 'value' => $transaction->getCategory()]); + $importJournal->setValue(['role' => 'note', 'value' => $notes]); + $importJournal->setValue(['role' => 'tags-comma', 'value' => join(',', $tags)]); + $collection->push($importJournal); + } + } + Log::debug(sprintf('Going to try and store all %d them.', $collection->count())); + // try to store them: + $storage = new ImportStorage; + + $storage->setJob($this->job); + $storage->setDateFormat('Y-m-d\TH:i:sO'); + $storage->setObjects($collection); + $storage->store(); + Log::info('Back in importTransactions()'); + + // link to tag + /** @var TagRepositoryInterface $repository */ + $repository = app(TagRepositoryInterface::class); + $repository->setUser($this->job->user); + $data = [ + 'tag' => trans('import.import_with_key', ['key' => $this->job->key]), + 'date' => new Carbon, + 'description' => null, + 'latitude' => null, + 'longitude' => null, + 'zoomLevel' => null, + 'tagMode' => 'nothing', + ]; + $tag = $repository->store($data); + $extended = $this->job->extended_status; + $extended['tag'] = $tag->id; + $this->job->extended_status = $extended; + $this->job->save(); + + Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag)); + Log::debug('Looping journals...'); + $journalIds = $storage->journals->pluck('id')->toArray(); + $tagId = $tag->id; + foreach ($journalIds as $journalId) { + Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId)); + DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]); + } + Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $storage->journals->count(), $tag->id, $tag->tag)); + + // set status to "finished"? + // update job: + $this->job->status = 'finished'; + $this->job->save(); + return; + } + + /** + * @throws FireflyException + * @throws SpectreException */ private function runStageHaveMapping() { - // for each spectre account id in 'account-mappings'. - // find FF account - // get transactions. - // import?! $config = $this->job->configuration; $accounts = $config['accounts'] ?? []; + $all = []; + $count = 0; /** @var array $accountArray */ foreach ($accounts as $accountArray) { $account = new Account($accountArray); $importId = intval($config['accounts-mapped'][$account->getid()] ?? 0); $doImport = $importId !== 0 ? true : false; if (!$doImport) { + Log::debug('Will NOT import from Spectre account #%d ("%s")', $account->getId(), $account->getName()); continue; } - // import into account + // grab all transactions $listTransactionsRequest = new ListTransactionsRequest($this->job->user); $listTransactionsRequest->setAccount($account); $listTransactionsRequest->call(); - $transactions = $listTransactionsRequest->getTransactions(); - var_dump($transactions);exit; + $transactions = $listTransactionsRequest->getTransactions(); + $all[$account->getId()] = [ + 'account' => $account, + 'import_id' => $importId, + 'transactions' => $transactions, + ]; + $count += count($transactions); + // add some steps done + $this->repository->addStepsDone($this->job, 2); } - var_dump($config); - exit; + // update number of steps: + $this->repository->setTotalSteps($this->job, $count * 5); + $this->repository->setStepsDone($this->job, 1); + Log::debug(sprintf('Total number of transactions: %d', $count)); + + + $this->importTransactions($all); } } diff --git a/app/Import/Storage/ImportStorage.php b/app/Import/Storage/ImportStorage.php index a9ff0341d0..3eb1da2d88 100644 --- a/app/Import/Storage/ImportStorage.php +++ b/app/Import/Storage/ImportStorage.php @@ -63,6 +63,7 @@ class ImportStorage private $matchBills = false; /** @var Collection */ private $objects; + private $total = 0; /** @var array */ private $transfers = []; @@ -116,6 +117,7 @@ class ImportStorage public function setObjects(Collection $objects) { $this->objects = $objects; + $this->total = $objects->count(); } /** @@ -150,7 +152,7 @@ class ImportStorage */ protected function storeImportJournal(int $index, ImportJournal $importJournal): bool { - Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $importJournal->getDescription())); + Log::debug(sprintf('Going to store object #%d/%d with description "%s"', ($index+1), $this->total, $importJournal->getDescription())); $assetAccount = $importJournal->asset->getAccount(); $amount = $importJournal->getAmount(); $currencyId = $this->getCurrencyId($importJournal); diff --git a/app/Models/ImportJob.php b/app/Models/ImportJob.php index 7dd51ff571..6447f96962 100644 --- a/app/Models/ImportJob.php +++ b/app/Models/ImportJob.php @@ -106,6 +106,7 @@ class ImportJob extends Model $status['done'] += $count; $this->extended_status = $status; $this->save(); + Log::debug(sprintf('Add %d to steps done for job "%s" making steps done %d', $count, $this->key, $status['done'])); } /** @@ -117,6 +118,7 @@ class ImportJob extends Model $status['steps'] += $count; $this->extended_status = $status; $this->save(); + Log::debug(sprintf('Add %d to total steps for job "%s" making total steps %d', $count, $this->key, $status['steps'])); } /** @@ -198,6 +200,30 @@ class ImportJob extends Model } } + /** + * @param int $steps + */ + public function setStepsDone(int $steps) + { + $status = $this->extended_status; + $status['done'] = $steps; + $this->extended_status = $status; + $this->save(); + Log::debug(sprintf('Set steps done for job "%s" to %d', $this->key, $steps)); + } + + /** + * @param int $count + */ + public function setTotalSteps(int $count) + { + $status = $this->extended_status; + $status['steps'] = $count; + $this->extended_status = $status; + $this->save(); + Log::debug(sprintf('Set total steps for job "%s" to %d', $this->key, $count)); + } + /** * @return string * diff --git a/app/Repositories/ImportJob/ImportJobRepository.php b/app/Repositories/ImportJob/ImportJobRepository.php index 7300ee4bc2..21d2156b13 100644 --- a/app/Repositories/ImportJob/ImportJobRepository.php +++ b/app/Repositories/ImportJob/ImportJobRepository.php @@ -311,4 +311,30 @@ class ImportJobRepository implements ImportJobRepositoryInterface { return $job->uploadFileContents(); } + + /** + * @param ImportJob $job + * @param int $count + * + * @return ImportJob + */ + public function setStepsDone(ImportJob $job, int $steps): ImportJob + { + $job->setStepsDone($steps); + + return $job; + } + + /** + * @param ImportJob $job + * @param int $count + * + * @return ImportJob + */ + public function setTotalSteps(ImportJob $job, int $count): ImportJob + { + $job->setTotalSteps($count); + + return $job; + } } diff --git a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php index 658befe70a..1834901226 100644 --- a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php +++ b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php @@ -31,6 +31,7 @@ use Symfony\Component\HttpFoundation\File\UploadedFile; */ interface ImportJobRepositoryInterface { + /** * @param ImportJob $job * @param int $steps @@ -112,6 +113,22 @@ interface ImportJobRepositoryInterface */ public function setExtendedStatus(ImportJob $job, array $array): ImportJob; + /** + * @param ImportJob $job + * @param int $count + * + * @return ImportJob + */ + public function setStepsDone(ImportJob $job, int $steps): ImportJob; + + /** + * @param ImportJob $job + * @param int $count + * + * @return ImportJob + */ + public function setTotalSteps(ImportJob $job, int $count): ImportJob; + /** * @param User $user */ diff --git a/app/Services/Spectre/Object/Account.php b/app/Services/Spectre/Object/Account.php index 92f01ed6c3..d904b119c4 100644 --- a/app/Services/Spectre/Object/Account.php +++ b/app/Services/Spectre/Object/Account.php @@ -78,6 +78,14 @@ class Account extends SpectreObject return $this->id; } + /** + * @return string + */ + public function getName(): string + { + return $this->name; + } + /** * @return array */ diff --git a/app/Services/Spectre/Object/Transaction.php b/app/Services/Spectre/Object/Transaction.php index 603c1777c7..5aa7cd5052 100644 --- a/app/Services/Spectre/Object/Transaction.php +++ b/app/Services/Spectre/Object/Transaction.php @@ -57,6 +57,14 @@ class Transaction extends SpectreObject /** @var Carbon */ private $updatedAt; + /** + * @return int + */ + public function getId(): int + { + return $this->id; + } + /** * Transaction constructor. * @@ -79,6 +87,39 @@ class Transaction extends SpectreObject $this->updatedAt = new Carbon($data['updated_at']); } + /** + * @return string + */ + public function getMode(): string + { + return $this->mode; + } + + /** + * @return string + */ + public function getStatus(): string + { + return $this->status; + } + + /** + * @return bool + */ + public function isDuplicated(): bool + { + return $this->duplicated; + } + + /** + * @return TransactionExtra + */ + public function getExtra(): TransactionExtra + { + return $this->extra; + } + + /** * @return string */ diff --git a/app/Services/Spectre/Object/TransactionExtra.php b/app/Services/Spectre/Object/TransactionExtra.php index 3874b5f8a8..6facc4af54 100644 --- a/app/Services/Spectre/Object/TransactionExtra.php +++ b/app/Services/Spectre/Object/TransactionExtra.php @@ -161,6 +161,7 @@ class TransactionExtra extends SpectreObject 'categorization_confidence' => $this->categorizationConfidence, ]; + return $array; }