diff --git a/app/Export/Collector/UploadCollector.php b/app/Export/Collector/UploadCollector.php index 17f863e721..a27de79610 100644 --- a/app/Export/Collector/UploadCollector.php +++ b/app/Export/Collector/UploadCollector.php @@ -15,6 +15,7 @@ namespace FireflyIII\Export\Collector; use Crypt; use Illuminate\Contracts\Encryption\DecryptException; +use Illuminate\Contracts\Filesystem\FileNotFoundException; use Log; use Storage; @@ -50,14 +51,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface public function run(): bool { Log::debug('Going to collect attachments', ['key' => $this->job->key]); - - // file names associated with the old import routine. - $this->vintageFormat = sprintf('csv-upload-%d-', $this->job->user->id); - - // collect old upload files (names beginning with "csv-upload". - $this->collectVintageUploads(); - - // then collect current upload files: $this->collectModernUploads(); return true; @@ -70,7 +63,8 @@ class UploadCollector extends BasicCollector implements CollectorInterface */ private function collectModernUploads(): bool { - $set = $this->job->user->importJobs()->where('status', 'import_complete')->get(['import_jobs.*']); + $set = $this->job->user->importJobs()->whereIn('status', ['import_complete', 'finished'])->get(['import_jobs.*']); + Log::debug(sprintf('Found %d import jobs', $set->count())); $keys = []; if ($set->count() > 0) { $keys = $set->pluck('key')->toArray(); @@ -83,59 +77,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface return true; } - /** - * This method collects all the uploads that are uploaded using the "old" importer. So from before the summer of 2016. - * - * @return bool - */ - private function collectVintageUploads(): bool - { - // grab upload directory. - $files = $this->uploadDisk->files(); - - foreach ($files as $entry) { - $this->processVintageUpload($entry); - } - - return true; - } - - /** - * This method tells you when the vintage upload file was actually uploaded. - * - * @param string $entry - * - * @return string - */ - private function getVintageUploadDate(string $entry): string - { - // this is an original upload. - $parts = explode('-', str_replace(['.csv.encrypted', $this->vintageFormat], '', $entry)); - $originalUpload = intval($parts[1]); - $date = date('Y-m-d \a\t H-i-s', $originalUpload); - - return $date; - } - - /** - * Tells you if a file name is a vintage upload. - * - * @param string $entry - * - * @return bool - */ - private function isVintageImport(string $entry): bool - { - $len = strlen($this->vintageFormat); - // file is part of the old import routine: - if (substr($entry, 0, $len) === $this->vintageFormat) { - - return true; - } - - return false; - } - /** * @param string $key * @@ -153,7 +94,7 @@ class UploadCollector extends BasicCollector implements CollectorInterface $content = ''; try { $content = Crypt::decrypt($this->uploadDisk->get(sprintf('%s.upload', $key))); - } catch (DecryptException $e) { + } catch (FileNotFoundException | DecryptException $e) { Log::error(sprintf('Could not decrypt old import file "%s". Skipped because: %s', $key, $e->getMessage())); } @@ -168,47 +109,4 @@ class UploadCollector extends BasicCollector implements CollectorInterface return true; } - /** - * If the file is a vintage upload, process it. - * - * @param string $entry - * - * @return bool - */ - private function processVintageUpload(string $entry): bool - { - if ($this->isVintageImport($entry)) { - $this->saveVintageImportFile($entry); - - return true; - } - - return false; - } - - - /** - * This will store the content of the old vintage upload somewhere. - * - * @param string $entry - */ - private function saveVintageImportFile(string $entry) - { - $content = ''; - try { - $content = Crypt::decrypt($this->uploadDisk->get($entry)); - } catch (DecryptException $e) { - Log::error('Could not decrypt old CSV import file ' . $entry . '. Skipped because ' . $e->getMessage()); - } - - if (strlen($content) > 0) { - // add to export disk. - $date = $this->getVintageUploadDate($entry); - $file = $this->job->key . '-Old import dated ' . $date . '.csv'; - $this->exportDisk->put($file, $content); - $this->getEntries()->push($file); - } - } - - } diff --git a/app/Export/Entry/Entry.php b/app/Export/Entry/Entry.php index 81ef189d36..817b71ce43 100644 --- a/app/Export/Entry/Entry.php +++ b/app/Export/Entry/Entry.php @@ -13,6 +13,7 @@ declare(strict_types=1); namespace FireflyIII\Export\Entry; +use FireflyIII\Models\Transaction; use Steam; /** @@ -37,24 +38,43 @@ final class Entry { // @formatter:off public $journal_id; + public $transaction_id = 0; + public $date; public $description; public $currency_code; public $amount; + public $foreign_currency_code = ''; + public $foreign_amount = '0'; public $transaction_type; public $asset_account_id; public $asset_account_name; + public $asset_account_iban; + public $asset_account_bic; + public $asset_account_number; + public $asset_currency_code; public $opposing_account_id; public $opposing_account_name; + public $opposing_account_iban; + public $opposing_account_bic; + public $opposing_account_number; + public $opposing_currency_code; public $budget_id; public $budget_name; + public $category_id; public $category_name; + + public $bill_id; + public $bill_name; + + public $notes; + public $tags; // @formatter:on /** @@ -95,5 +115,72 @@ final class Entry return $entry; } + /** + * Converts a given transaction (as collected by the collector) into an export entry. + * + * @param Transaction $transaction + * + * @return Entry + */ + public static function fromTransaction(Transaction $transaction): Entry + { + $entry = new self; + $entry->journal_id = $transaction->journal_id; + $entry->transaction_id = $transaction->id; + $entry->date = $transaction->date->format('Ymd'); + $entry->description = $transaction->description; + if (strlen(strval($transaction->transaction_description)) > 0) { + $entry->description = $transaction->transaction_description . '(' . $transaction->description . ')'; + } + $entry->currency_code = $transaction->transactionCurrency->code; + $entry->amount = round($transaction->transaction_amount, $transaction->transactionCurrency->decimal_places); + + $entry->foreign_currency_code = is_null($transaction->foreign_currency_id) ? null : $transaction->foreignCurrency->code; + $entry->foreign_amount = is_null($transaction->foreign_currency_id) + ? null + : round( + $transaction->transaction_foreign_amount, $transaction->foreignCurrency->decimal_places + ); + + $entry->transaction_type = $transaction->transaction_type_type; + $entry->asset_account_id = $transaction->account_id; + $entry->asset_account_name = app('steam')->tryDecrypt($transaction->account_name); + $entry->asset_account_iban = $transaction->account_iban; + $entry->asset_account_number = $transaction->account_number; + $entry->asset_account_bic = $transaction->account_bic; + // asset_currency_code + $entry->opposing_account_id = $transaction->opposing_account_id; + $entry->opposing_account_name = app('steam')->tryDecrypt($transaction->opposing_account_name); + $entry->opposing_account_iban = $transaction->opposing_account_iban; + $entry->opposing_account_number = $transaction->opposing_account_number; + $entry->opposing_account_bic = $transaction->opposing_account_bic; + // opposing currency code + + /** budget */ + $entry->budget_id = $transaction->transaction_budget_id; + $entry->budget_name = app('steam')->tryDecrypt($transaction->transaction_budget_name); + if (is_null($transaction->transaction_budget_id)) { + $entry->budget_id = $transaction->transaction_journal_budget_id; + $entry->budget_name = app('steam')->tryDecrypt($transaction->transaction_journal_budget_name); + } + + /** category */ + $entry->category_id = $transaction->transaction_category_id; + $entry->category_name = app('steam')->tryDecrypt($transaction->transaction_category_name); + if (is_null($transaction->transaction_category_id)) { + $entry->category_id = $transaction->transaction_journal_category_id; + $entry->category_name = app('steam')->tryDecrypt($transaction->transaction_journal_category_name); + } + + /** budget */ + $entry->bill_id = $transaction->bill_id; + $entry->bill_name = app('steam')->tryDecrypt($transaction->bill_name); + + $entry->tags = $transaction->tags; + $entry->notes = $transaction->notes; + + return $entry; + } + } diff --git a/app/Export/ExpandedProcessor.php b/app/Export/ExpandedProcessor.php new file mode 100644 index 0000000000..703963f27d --- /dev/null +++ b/app/Export/ExpandedProcessor.php @@ -0,0 +1,308 @@ +journals = new Collection; + $this->exportEntries = new Collection; + $this->files = new Collection; + } + + /** + * @return bool + */ + public function collectAttachments(): bool + { + /** @var AttachmentCollector $attachmentCollector */ + $attachmentCollector = app(AttachmentCollector::class); + $attachmentCollector->setJob($this->job); + $attachmentCollector->setDates($this->settings['startDate'], $this->settings['endDate']); + $attachmentCollector->run(); + $this->files = $this->files->merge($attachmentCollector->getEntries()); + + return true; + } + + /** + * @return bool + */ + public function collectJournals(): bool + { + // use journal collector thing. + /** @var JournalCollectorInterface $collector */ + $collector = app(JournalCollectorInterface::class); + $collector->setAccounts($this->accounts)->setRange($this->settings['startDate'], $this->settings['endDate']) + ->withOpposingAccount()->withBudgetInformation()->withCategoryInformation() + ->removeFilter(InternalTransferFilter::class); + $transactions = $collector->getJournals(); + // get some more meta data for each entry: + $ids = $transactions->pluck('journal_id')->toArray(); + $assetIds = $transactions->pluck('account_id')->toArray(); + $opposingIds = $transactions->pluck('opposing_account_id')->toArray(); + $notes = $this->getNotes($ids); + $tags = $this->getTags($ids); + $ibans = $this->getIbans($assetIds) + $this->getIbans($opposingIds); + $transactions->each( + function (Transaction $transaction) use ($notes, $tags, $ibans) { + $journalId = intval($transaction->journal_id); + $accountId = intval($transaction->account_id); + $opposingId = intval($transaction->opposing_account_id); + $transaction->notes = $notes[$journalId] ?? ''; + $transaction->tags = join(',', $tags[$journalId] ?? []); + $transaction->account_number = $ibans[$accountId]['accountNumber'] ?? ''; + $transaction->account_bic = $ibans[$accountId]['BIC'] ?? ''; + $transaction->opposing_account_number = $ibans[$opposingId]['accountNumber'] ?? ''; + $transaction->opposing_account_bic = $ibans[$opposingId]['BIC'] ?? ''; + + } + ); + + $this->journals = $transactions; + + return true; + } + + /** + * @return bool + */ + public function collectOldUploads(): bool + { + /** @var UploadCollector $uploadCollector */ + $uploadCollector = app(UploadCollector::class); + $uploadCollector->setJob($this->job); + $uploadCollector->run(); + + $this->files = $this->files->merge($uploadCollector->getEntries()); + + return true; + } + + /** + * @return bool + */ + public function convertJournals(): bool + { + $this->journals->each( + function (Transaction $transaction) { + $this->exportEntries->push(Entry::fromTransaction($transaction)); + } + ); + Log::debug(sprintf('Count %d entries in exportEntries (convertJournals)', $this->exportEntries->count())); + + return true; + } + + /** + * @return bool + * @throws FireflyException + */ + public function createZipFile(): bool + { + $zip = new ZipArchive; + $file = $this->job->key . '.zip'; + $fullPath = storage_path('export') . '/' . $file; + + if ($zip->open($fullPath, ZipArchive::CREATE) !== true) { + throw new FireflyException('Cannot store zip file.'); + } + // for each file in the collection, add it to the zip file. + $disk = Storage::disk('export'); + foreach ($this->getFiles() as $entry) { + // is part of this job? + $zipFileName = str_replace($this->job->key . '-', '', $entry); + $zip->addFromString($zipFileName, $disk->get($entry)); + } + + $zip->close(); + + // delete the files: + $this->deleteFiles(); + + return true; + } + + /** + * @return bool + */ + public function exportJournals(): bool + { + $exporterClass = config('firefly.export_formats.' . $this->exportFormat); + $exporter = app($exporterClass); + $exporter->setJob($this->job); + $exporter->setEntries($this->exportEntries); + $exporter->run(); + $this->files->push($exporter->getFileName()); + + return true; + } + + /** + * @return Collection + */ + public function getFiles(): Collection + { + return $this->files; + } + + /** + * Save export job settings to class. + * + * @param array $settings + */ + public function setSettings(array $settings) + { + // save settings + $this->settings = $settings; + $this->accounts = $settings['accounts']; + $this->exportFormat = $settings['exportFormat']; + $this->includeAttachments = $settings['includeAttachments']; + $this->includeOldUploads = $settings['includeOldUploads']; + $this->job = $settings['job']; + } + + /** + * + */ + private function deleteFiles() + { + $disk = Storage::disk('export'); + foreach ($this->getFiles() as $file) { + $disk->delete($file); + } + } + + /** + * Get all IBAN / SWIFT / account numbers + * + * @param array $array + * + * @return array + */ + private function getIbans(array $array): array + { + $array = array_unique($array); + $return = []; + $set = AccountMeta::whereIn('account_id', $array) + ->leftJoin('accounts', 'accounts.id', 'account_meta.account_id') + ->where('accounts.user_id', $this->job->user_id) + ->whereIn('account_meta.name', ['accountNumber', 'BIC', 'currency_id']) + ->get(['account_meta.id', 'account_meta.account_id', 'account_meta.name', 'account_meta.data']); + /** @var AccountMeta $meta */ + foreach ($set as $meta) { + $id = intval($meta->account_id); + $return[$id][$meta->name] = $meta->data; + } + + return $return; + } + + /** + * Returns, if present, for the given journal ID's the notes. + * + * @param array $array + * + * @return array + */ + private function getNotes(array $array): array + { + $array = array_unique($array); + $set = TransactionJournalMeta::whereIn('journal_meta.transaction_journal_id', $array) + ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id') + ->where('transaction_journals.user_id', $this->job->user_id) + ->where('journal_meta.name', 'notes')->get( + ['journal_meta.transaction_journal_id', 'journal_meta.data', 'journal_meta.id'] + ); + $return = []; + /** @var TransactionJournalMeta $meta */ + foreach ($set as $meta) { + $id = intval($meta->transaction_journal_id); + $return[$id] = $meta->data; + } + + return $return; + } + + /** + * Returns a comma joined list of all the users tags linked to these journals. + * + * @param array $array + * + * @return array + */ + private function getTags(array $array): array + { + $set = DB::table('tag_transaction_journal') + ->whereIn('tag_transaction_journal.transaction_journal_id', $array) + ->leftJoin('tags', 'tag_transaction_journal.tag_id', '=', 'tags.id') + ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'tag_transaction_journal.transaction_journal_id') + ->where('transaction_journals.user_id', $this->job->user_id) + ->get(['tag_transaction_journal.transaction_journal_id', 'tags.tag']); + $result = []; + foreach ($set as $entry) { + $id = intval($entry->transaction_journal_id); + $result[$id] = isset($result[$id]) ? $result[$id] : []; + $result[$id][] = Crypt::decrypt($entry->tag); + } + + return $result; + } +} \ No newline at end of file diff --git a/app/Export/Exporter/CsvExporter.php b/app/Export/Exporter/CsvExporter.php index 34a56182e0..dea7519fa7 100644 --- a/app/Export/Exporter/CsvExporter.php +++ b/app/Export/Exporter/CsvExporter.php @@ -58,8 +58,12 @@ class CsvExporter extends BasicExporter implements ExporterInterface // get field names for header row: $first = $this->getEntries()->first(); - $headers = array_keys(get_object_vars($first)); - $rows[] = $headers; + $headers = []; + if (!is_null($first)) { + $headers = array_keys(get_object_vars($first)); + } + + $rows[] = $headers; /** @var Entry $entry */ foreach ($this->getEntries() as $entry) { diff --git a/app/Helpers/Collector/JournalCollector.php b/app/Helpers/Collector/JournalCollector.php index c31295c235..d892cefdf8 100644 --- a/app/Helpers/Collector/JournalCollector.php +++ b/app/Helpers/Collector/JournalCollector.php @@ -31,7 +31,6 @@ use FireflyIII\Models\Tag; use FireflyIII\Models\Transaction; use FireflyIII\Repositories\Account\AccountRepositoryInterface; use FireflyIII\User; -use Illuminate\Contracts\Encryption\DecryptException; use Illuminate\Database\Eloquent\Builder as EloquentBuilder; use Illuminate\Database\Query\JoinClause; use Illuminate\Pagination\LengthAwarePaginator; @@ -86,7 +85,9 @@ class JournalCollector implements JournalCollectorInterface 'accounts.name as account_name', 'accounts.encrypted as account_encrypted', + 'accounts.iban as account_iban', 'account_types.type as account_type', + ]; /** @var bool */ private $filterTransfers = false; @@ -175,12 +176,10 @@ class JournalCollector implements JournalCollectorInterface if (!is_null($transaction->bill_name)) { $transaction->bill_name = Steam::decrypt(intval($transaction->bill_name_encrypted), $transaction->bill_name); } + $transaction->opposing_account_name = app('steam')->tryDecrypt($transaction->opposing_account_name); + $transaction->account_iban = app('steam')->tryDecrypt($transaction->account_iban); + $transaction->opposing_account_iban = app('steam')->tryDecrypt($transaction->opposing_account_iban); - try { - $transaction->opposing_account_name = Crypt::decrypt($transaction->opposing_account_name); - } catch (DecryptException $e) { - // if this fails its already decrypted. - } } ); @@ -677,10 +676,12 @@ class JournalCollector implements JournalCollectorInterface $this->query->leftJoin('account_types as opposing_account_types', 'opposing_accounts.account_type_id', '=', 'opposing_account_types.id'); $this->query->whereNull('opposing.deleted_at'); - $this->fields[] = 'opposing.id as opposing_id'; - $this->fields[] = 'opposing.account_id as opposing_account_id'; - $this->fields[] = 'opposing_accounts.name as opposing_account_name'; - $this->fields[] = 'opposing_accounts.encrypted as opposing_account_encrypted'; + $this->fields[] = 'opposing.id as opposing_id'; + $this->fields[] = 'opposing.account_id as opposing_account_id'; + $this->fields[] = 'opposing_accounts.name as opposing_account_name'; + $this->fields[] = 'opposing_accounts.encrypted as opposing_account_encrypted'; + $this->fields[] = 'opposing_accounts.iban as opposing_account_iban'; + $this->fields[] = 'opposing_account_types.type as opposing_account_type'; $this->joinedOpposing = true; Log::debug('joinedOpposing is now true!'); diff --git a/app/Http/Controllers/ExportController.php b/app/Http/Controllers/ExportController.php index 6bb4cd3f44..326ad87c77 100644 --- a/app/Http/Controllers/ExportController.php +++ b/app/Http/Controllers/ExportController.php @@ -17,6 +17,7 @@ namespace FireflyIII\Http\Controllers; use Carbon\Carbon; use ExpandedForm; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Export\ExpandedProcessor; use FireflyIII\Export\ProcessorInterface; use FireflyIII\Http\Requests\ExportFormRequest; use FireflyIII\Models\AccountType; @@ -137,34 +138,40 @@ class ExportController extends Controller public function postIndex(ExportFormRequest $request, AccountRepositoryInterface $repository, ExportJobRepositoryInterface $jobs) { $job = $jobs->findByKey($request->get('job')); + $accounts = $request->get('accounts') ?? []; $settings = [ - 'accounts' => $repository->getAccountsById($request->get('accounts')), + 'accounts' => $repository->getAccountsById($accounts), 'startDate' => new Carbon($request->get('export_start_range')), 'endDate' => new Carbon($request->get('export_end_range')), 'exportFormat' => $request->get('exportFormat'), - 'includeAttachments' => intval($request->get('include_attachments')) === 1, - 'includeOldUploads' => intval($request->get('include_old_uploads')) === 1, + 'includeAttachments' => $request->boolean('include_attachments'), + 'includeOldUploads' => $request->boolean('include_old_uploads'), 'job' => $job, ]; $jobs->changeStatus($job, 'export_status_make_exporter'); /** @var ProcessorInterface $processor */ - $processor = app(ProcessorInterface::class); + $processor = app(ExpandedProcessor::class); $processor->setSettings($settings); + + + /* * Collect journals: */ $jobs->changeStatus($job, 'export_status_collecting_journals'); $processor->collectJournals(); $jobs->changeStatus($job, 'export_status_collected_journals'); + /* * Transform to exportable entries: */ $jobs->changeStatus($job, 'export_status_converting_to_export_format'); $processor->convertJournals(); $jobs->changeStatus($job, 'export_status_converted_to_export_format'); + /* * Transform to (temporary) file: */ @@ -180,6 +187,7 @@ class ExportController extends Controller $jobs->changeStatus($job, 'export_status_collected_attachments'); } + /* * Collect old uploads */ diff --git a/app/Http/Requests/ExportFormRequest.php b/app/Http/Requests/ExportFormRequest.php index 16a5638302..fac7f1ee2c 100644 --- a/app/Http/Requests/ExportFormRequest.php +++ b/app/Http/Requests/ExportFormRequest.php @@ -38,20 +38,19 @@ class ExportFormRequest extends Request public function rules() { $sessionFirst = clone session('first'); - - $first = $sessionFirst->subDay()->format('Y-m-d'); - $today = Carbon::create()->addDay()->format('Y-m-d'); - $formats = join(',', array_keys(config('firefly.export_formats'))); + $first = $sessionFirst->subDay()->format('Y-m-d'); + $today = Carbon::create()->addDay()->format('Y-m-d'); + $formats = join(',', array_keys(config('firefly.export_formats'))); return [ - 'export_start_range' => 'required|date|after:' . $first, - 'export_end_range' => 'required|date|before:' . $today, - 'accounts' => 'required', - 'job' => 'required|belongsToUser:export_jobs,key', - 'accounts.*' => 'required|exists:accounts,id|belongsToUser:accounts', - 'include_attachments' => 'in:0,1', - 'include_config' => 'in:0,1', - 'exportFormat' => 'in:' . $formats, +// 'export_start_range' => 'required|date|after:' . $first, +// 'export_end_range' => 'required|date|before:' . $today, +// 'accounts' => 'required', +// 'job' => 'required|belongsToUser:export_jobs,key', +// 'accounts.*' => 'required|exists:accounts,id|belongsToUser:accounts', +// 'include_attachments' => 'in:0,1', +// 'include_config' => 'in:0,1', +// 'exportFormat' => 'in:' . $formats, ]; } } diff --git a/app/Http/Requests/Request.php b/app/Http/Requests/Request.php index 16e77bad4a..87e1bf2934 100644 --- a/app/Http/Requests/Request.php +++ b/app/Http/Requests/Request.php @@ -28,7 +28,7 @@ class Request extends FormRequest * * @return bool */ - protected function boolean(string $field): bool + public function boolean(string $field): bool { return intval($this->input($field)) === 1; } diff --git a/app/Models/Transaction.php b/app/Models/Transaction.php index c9b139d485..2b9deae872 100644 --- a/app/Models/Transaction.php +++ b/app/Models/Transaction.php @@ -22,6 +22,42 @@ use Watson\Validating\ValidatingTrait; /** * Class Transaction * + * @property-read int $journal_id + * @property-read Carbon $date + * @property-read string $transaction_description + * @property-read string $transaction_amount + * @property-read string $transaction_foreign_amount + * @property-read string $transaction_type_type + * + * @property-read int $account_id + * @property-read string $account_name + * @property string $account_iban + * @property string $account_number + * @property string $account_bic + * + * @property-read int $opposing_account_id + * @property string $opposing_account_name + * @property string $opposing_account_iban + * @property string $opposing_account_number + * @property string $opposing_account_bic + * + * + * @property-read int $transaction_budget_id + * @property-read string $transaction_budget_name + * @property-read int $transaction_journal_budget_id + * @property-read string $transaction_journal_budget_name + * + * @property-read int $transaction_category_id + * @property-read string $transaction_category_name + * @property-read int $transaction_journal_category_id + * @property-read string $transaction_journal_category_name + * + * @property-read int $bill_id + * @property string $bill_name + * + * @property string $notes + * @property string $tags + * * @package FireflyIII\Models */ class Transaction extends Model diff --git a/resources/views/export/index.twig b/resources/views/export/index.twig index 3ebcc538ea..2f7aaba776 100644 --- a/resources/views/export/index.twig +++ b/resources/views/export/index.twig @@ -8,7 +8,7 @@ -