IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

Can configure file upload in file imports.

Browse Source
This commit is contained in:
James Cole
2018-05-06 07:09:08 +02:00
parent f74b9ba7ab
commit 7d80ac37a6
19 changed files with 791 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/Http/Controllers/Import/JobConfigurationController.php
View File

@@ -29,6 +29,8 @@ use FireflyIII\Import\JobConfiguration\JobConfigurationInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Http\Request;
use Illuminate\Http\UploadedFile;
use Illuminate\Support\MessageBag;
use Log;
/**
@@ -139,8 +141,17 @@ class JobConfigurationController extends Controller
return redirect(route('import.job.status.index', [$importJob->key]));
}
// uploaded files are attached to the job.
// the configurator can then handle them.
$result = new MessageBag;
/** @var UploadedFile $upload */
foreach ($request->allFiles() as $name => $upload) {
$result = $this->repository->storeFileUpload($importJob, $name, $upload);
}
$data = $request->all();
$messages = $configurator->configureJob($data);
$result->merge($messages);
if ($messages->count() > 0) {
$request->session()->flash('warning', $messages->first());

81
app/Import/JobConfiguration/FileJobConfiguration.php
View File

@@ -23,11 +23,20 @@ declare(strict_types=1);
namespace FireflyIII\Import\JobConfiguration;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Configuration\File\ConfigurationInterface;
use FireflyIII\Support\Import\Configuration\File\ConfigureUploadHandler;
use FireflyIII\Support\Import\Configuration\File\NewFileJobHandler;
use Illuminate\Support\MessageBag;
class FileJobConfiguration implements JobConfigurationInterface
{
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* ConfiguratorInterface constructor.
@@ -43,30 +52,86 @@ class FileJobConfiguration implements JobConfigurationInterface
* @param array $data
*
* @return MessageBag
* @throws FireflyException
*/
public function configureJob(array $data): MessageBag
{
// TODO: Implement configureJob() method.
$configurator = $this->getConfigurationObject();
$configurator->setJob($this->importJob);
return $configurator->configureJob($data);
}
/**
* Return the data required for the next step in the job configuration.
*
* @throws FireflyException
* @return array
*/
public function getNextData(): array
{
// TODO: Implement getNextData() method.
$configurator = $this->getConfigurationObject();
$configurator->setJob($this->importJob);
return $configurator->getNextData();
}
/**
* Get the configuration handler for this specific stage.
*
* @return ConfigurationInterface
* @throws FireflyException
*/
private function getConfigurationObject(): ConfigurationInterface
{
$class = 'DoNotExist';
switch ($this->importJob->stage) {
case 'new': // has nothing, no file upload or anything.
$class = NewFileJobHandler::class;
break;
case 'configure-upload':
$class = ConfigureUploadHandler::class;
break;
// case 'upload-config': // has file, needs file config.
// $class = UploadConfig::class;
// break;
// case 'roles': // has configured file, needs roles.
// $class = Roles::class;
// break;
// case 'map': // has roles, needs mapping.
// $class = Map::class;
// break;
// default:
// break;
}
if (!class_exists($class)) {
throw new FireflyException(sprintf('Class %s does not exist in getConfigurationClass().', $class)); // @codeCoverageIgnore
}
return app($class);
}
/**
* Returns the view of the next step in the job configuration.
*
* @throws FireflyException
* @return string
*/
public function getNextView(): string
{
// TODO: Implement getNextView() method.
switch ($this->importJob->stage) {
case 'new':
return 'import.file.new';
case 'configure-upload':
return 'import.file.configure-upload';
break;
default:
// @codeCoverageIgnoreStart
throw new FireflyException(
sprintf('FileJobConfiguration::getNextView() cannot handle stage "%s"', $this->importJob->stage)
);
// @codeCoverageIgnoreEnd
}
}
/**
@@ -76,7 +141,11 @@ class FileJobConfiguration implements JobConfigurationInterface
*/
public function configurationComplete(): bool
{
// TODO: Implement configurationComplete() method.
if ($this->importJob->stage === 'ready_to run') {
return true;
}
return false;
}
/**
@@ -84,6 +153,8 @@ class FileJobConfiguration implements JobConfigurationInterface
*/
public function setJob(ImportJob $job): void
{
// TODO: Implement setJob() method.
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
}

4
app/Import/Specifics/AbnAmroDescription.php
View File

@@ -41,7 +41,7 @@ class AbnAmroDescription implements SpecificInterface
*/
public static function getDescription(): string
{
return 'Fixes possible problems with ABN Amro descriptions.';
return 'import.specific_abn_descr';
}
/**
@@ -50,7 +50,7 @@ class AbnAmroDescription implements SpecificInterface
*/
public static function getName(): string
{
return 'ABN Amro description';
return 'import.specific_abn_name';
}
/**

4
app/Import/Specifics/IngDescription.php
View File

@@ -43,7 +43,7 @@ class IngDescription implements SpecificInterface
*/
public static function getDescription(): string
{
return 'Create better descriptions in ING import files.';
return 'import.specific_ing_descr';
}
/**
@@ -52,7 +52,7 @@ class IngDescription implements SpecificInterface
*/
public static function getName(): string
{
return 'ING description';
return 'import.specific_ing_name';
}
/**

4
app/Import/Specifics/PresidentsChoice.php
View File

@@ -33,7 +33,7 @@ class PresidentsChoice implements SpecificInterface
*/
public static function getDescription(): string
{
return 'Fixes problems with files from Presidents Choice Financial.';
return 'import.specific_pres_descr';
}
/**
@@ -42,7 +42,7 @@ class PresidentsChoice implements SpecificInterface
*/
public static function getName(): string
{
return 'Presidents "Choice"';
return 'import.specific_pres_name';
}
/**

4
app/Import/Specifics/RabobankDescription.php
View File

@@ -35,7 +35,7 @@ class RabobankDescription implements SpecificInterface
*/
public static function getDescription(): string
{
return 'Fixes possible problems with Rabobank descriptions.';
return 'import.specific_pres_descr';
}
/**
@@ -44,7 +44,7 @@ class RabobankDescription implements SpecificInterface
*/
public static function getName(): string
{
return 'Rabobank description';
return 'import.specific_rabo_name';
}
/**

4
app/Import/Specifics/SnsDescription.php
View File

@@ -33,7 +33,7 @@ class SnsDescription implements SpecificInterface
*/
public static function getDescription(): string
{
return 'Trim quotes from SNS descriptions.';
return 'import.specific_sns_descr';
}
/**
@@ -42,7 +42,7 @@ class SnsDescription implements SpecificInterface
*/
public static function getName(): string
{
return 'SNS description';
return 'import.specific_sns_name';
}
/**

9
app/Models/ImportJob.php
View File

@@ -52,6 +52,15 @@ class ImportJob extends Model
/** @var array */
protected $fillable = ['key', 'user_id', 'file_type', 'provider', 'status', 'stage', 'configuration', 'extended_status', 'transactions', 'errors'];
/**
* @codeCoverageIgnore
* @return \Illuminate\Database\Eloquent\Relations\MorphMany
*/
public function attachments()
{
return $this->morphMany(Attachment::class, 'attachable');
}
/**
* @param $value
*

84
app/Repositories/ImportJob/ImportJobRepository.php
View File

@@ -24,11 +24,13 @@ namespace FireflyIII\Repositories\ImportJob;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use FireflyIII\User;
use Illuminate\Support\MessageBag;
use Illuminate\Support\Str;
use Log;
use SplFileObject;
@@ -42,6 +44,16 @@ class ImportJobRepository implements ImportJobRepositoryInterface
{
/** @var User */
private $user;
/** @var int */
private $maxUploadSize;
/** @var \Illuminate\Contracts\Filesystem\Filesystem */
protected $uploadDisk;
public function __construct()
{
$this->maxUploadSize = (int)config('firefly.maxUploadSize');
$this->uploadDisk = Storage::disk('upload');
}
/**
* @param ImportJob $job
@@ -421,6 +433,8 @@ class ImportJobRepository implements ImportJobRepositoryInterface
/**
* Return import file content.
*
* @deprecated
*
* @param ImportJob $job
*
* @return string
@@ -476,4 +490,74 @@ class ImportJobRepository implements ImportJobRepositoryInterface
return $job;
}
/**
* @codeCoverageIgnore
*
* @param UploadedFile $file
*
* @return bool
*/
protected function validSize(UploadedFile $file): bool
{
$size = $file->getSize();
return $size > $this->maxUploadSize;
}
/**
* Handle upload for job.
*
* @param ImportJob $job
* @param string $name
* @param UploadedFile $file
*
* @return MessageBag
* @throws FireflyException
*/
public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag
{
$messages = new MessageBag;
if ($this->validSize($file)) {
$name = e($file->getClientOriginalName());
$messages->add('size', (string)trans('validation.file_too_large', ['name' => $name]));
return $messages;
}
$count = $job->attachments()->get()->filter(
function (Attachment $att) use($name) {
return $att->filename === $name;
}
)->count();
if ($count > 0) {
// don't upload, but also don't complain about it.
Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name));
return new MessageBag;
}
$attachment = new Attachment; // create Attachment object.
$attachment->user()->associate($job->user);
$attachment->attachable()->associate($job);
$attachment->md5 = md5_file($file->getRealPath());
$attachment->filename = $name;
$attachment->mime = $file->getMimeType();
$attachment->size = $file->getSize();
$attachment->uploaded = 0;
$attachment->save();
$fileObject = $file->openFile('r');
$fileObject->rewind();
$content = $fileObject->fread($file->getSize());
$encrypted = Crypt::encrypt($content);
// store it:
$this->uploadDisk->put($attachment->fileName(), $encrypted);
$attachment->uploaded = 1; // update attachment
$attachment->save();
// return it.
return new MessageBag;
}
}

14
app/Repositories/ImportJob/ImportJobRepositoryInterface.php
View File

@@ -22,9 +22,11 @@ declare(strict_types=1);
namespace FireflyIII\Repositories\ImportJob;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\User;
use Illuminate\Support\MessageBag;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
@@ -33,6 +35,18 @@ use Symfony\Component\HttpFoundation\File\UploadedFile;
interface ImportJobRepositoryInterface
{
/**
* Handle upload for job.
*
* @param ImportJob $job
* @param string $name
* @param UploadedFile $file
*
* @return MessageBag
* @throws FireflyException
*/
public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag;
/**
* @param ImportJob $job
* @param array $transactions

11
app/Support/ExpandedForm.php
View File

@@ -145,16 +145,6 @@ class ExpandedForm
*/
public function assetAccountList(string $name, $value = null, array $options = []): string
{
// properties for cache
$cache = new CacheProperties;
$cache->addProperty('exp-form-asset-list');
$cache->addProperty($name);
$cache->addProperty($value);
$cache->addProperty($options);
if ($cache->has()) {
return $cache->get();
}
// make repositories
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
@@ -182,7 +172,6 @@ class ExpandedForm
$grouped[$key][$account->id] = $account->name . ' (' . app('amount')->formatAnything($currency, $balance, false) . ')';
}
$res = $this->select($name, $grouped, $value, $options);
$cache->store($res);
return $res;
}

19
app/Support/Import/Configuration/ConfigurationInterface.php → app/Support/Import/Configuration/File/ConfigurationInterface.php
View File

@@ -20,12 +20,12 @@
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration;
namespace FireflyIII\Support\Import\Configuration\File;
use FireflyIII\Models\ImportJob;
use Illuminate\Support\MessageBag;
/**
* @deprecated
* Class ConfigurationInterface.
*/
interface ConfigurationInterface
@@ -35,14 +35,7 @@ interface ConfigurationInterface
*
* @return array
*/
public function getData(): array;
/**
* Return possible warning to user.
*
* @return string
*/
public function getWarningMessage(): string;
public function getNextData(): array;
/**
* @param ImportJob $job
@@ -52,11 +45,11 @@ interface ConfigurationInterface
public function setJob(ImportJob $job);
/**
* Store the result.
* Store data associated with current stage.
*
* @param array $data
*
* @return bool
* @return MessageBag
*/
public function storeConfiguration(array $data): bool;
public function configureJob(array $data): MessageBag;
}

167
app/Support/Import/Configuration/File/ConfigureUploadHandler.php Normal file
View File

@@ -0,0 +1,167 @@
<?php
/**
* ConfigureUploadHandlerphp
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\File;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Support\MessageBag;
use Log;
/**
* Class ConfigureUploadHandler
*
* @package FireflyIII\Support\Import\Configuration\File
*/
class ConfigureUploadHandler implements ConfigurationInterface
{
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var AccountRepositoryInterface */
private $accountRepos;
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$delimiters = [
',' => trans('form.csv_comma'),
';' => trans('form.csv_semicolon'),
'tab' => trans('form.csv_tab'),
];
$config = $this->importJob->configuration;
$config['date-format'] = $config['date-format'] ?? 'Ymd';
$specifics = [];
$this->repository->setConfiguration($this->importJob, $config);
// collect specifics.
foreach (config('csv.import_specifics') as $name => $className) {
$specifics[$name] = [
'name' => $className::getName(),
'description' => $className::getDescription(),
];
}
$data = [
'accounts' => [],
'specifix' => [],
'delimiters' => $delimiters,
'specifics' => $specifics,
];
return $data;
}
/**
* @param ImportJob $job
*
* @return ConfigurationInterface
*/
public function setJob(ImportJob $job)
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
$this->accountRepos = app(AccountRepositoryInterface::class);
$this->accountRepos->setUser($job->user);
}
/**
* Store data associated with current stage.
*
* @param array $data
*
* @return MessageBag
*/
public function configureJob(array $data): MessageBag
{
$config = $this->importJob->configuration;
$complete = true;
// collect values:
$importId = isset($data['csv_import_account']) ? (int)$data['csv_import_account'] : 0;
$delimiter = (string)$data['csv_delimiter'];
$config['has-headers'] = (int)($data['has_headers'] ?? 0.0) === 1;
$config['date-format'] = (string)$data['date_format'];
$config['delimiter'] = 'tab' === $delimiter ? "\t" : $delimiter;
$config['apply-rules'] = (int)($data['apply_rules'] ?? 0.0) === 1;
$config['specifics'] = $this->getSpecifics($data);
// validate values:
$account = $this->accountRepos->findNull($importId);
// respond to invalid account:
if (null === $account) {
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
$complete = false;
}
if (null !== $account) {
$config['import-account'] = $account->id;
}
$this->repository->setConfiguration($this->importJob, $config);
if ($complete) {
$this->repository->setStage($this->importJob, 'roles');
}
if (!$complete) {
$messages = new MessageBag;
$messages->add('account', trans('import.invalid_import_account'));
return $messages;
}
return new MessageBag;
}
/**
* @param array $data
*
* @return array
*/
private function getSpecifics(array $data): array
{
$return = [];
// check if specifics given are correct:
if (isset($data['specifics']) && \is_array($data['specifics'])) {
foreach ($data['specifics'] as $name) {
// verify their content.
$className = sprintf('FireflyIII\Import\Specifics\%s', $name);
if (class_exists($className)) {
$return[$name] = 1;
}
}
}
return $return;
}
}

176
app/Support/Import/Configuration/File/NewFileJobHandler.php Normal file
View File

@@ -0,0 +1,176 @@
<?php
/**
* NewFileJobHandler.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\File;
use Crypt;
use FireflyIII\Console\Commands\Import;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Contracts\Encryption\DecryptException;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\MessageBag;
use Log;
use Storage;
use Exception;
/**
* Class NewFileJobHandler
*
* @package FireflyIII\Support\Import\Configuration\File
*/
class NewFileJobHandler implements ConfigurationInterface
{
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* Store data associated with current stage.
*
* @param array $data
*
* @throws FireflyException
* @return MessageBag
*/
public function configureJob(array $data): MessageBag
{
// nothing to store, validate upload
// and push to next stage.
$messages = new MessageBag;
$attachments = $this->importJob->attachments;
/** @var Attachment $attachment */
foreach ($attachments as $attachment) {
// check if content is UTF8:
if (!$this->isUTF8($attachment)) {
$message = trans('import.file_not_utf8');
Log::error($message);
$messages->add('import_file', $message);
// delete attachment:
try {
$attachment->delete();
} catch (Exception $e) {
throw new FireflyException(sprintf('Could not delete attachment: %s', $e->getMessage()));
}
return $messages;
}
// if file is configuration file, store it into the job.
if ($attachment->filename === 'configuration_file') {
$this->storeConfig($attachment);
}
}
$this->repository->setStage($this->importJob, 'configure-upload');
return new MessageBag();
}
/**
* @param Attachment $attachment
*
* @return bool
* @throws FireflyException
*/
private function isUTF8(Attachment $attachment): bool
{
$disk = Storage::disk('upload');
try {
$content = $disk->get(sprintf('at-%d.data', $attachment->id));
$content = Crypt::decrypt($content);
} catch (FileNotFoundException|DecryptException $e) {
Log::error($e->getMessage());
throw new FireflyException($e->getMessage());
}
$result = mb_detect_encoding($content, 'UTF-8', true);
if ($result === false) {
return false;
}
if ($result !== 'ASCII' && $result !== 'UTF-8') {
return false;
}
return true;
}
/**
* @param Attachment $attachment
*
* @throws FireflyException
*/
private function storeConfig(Attachment $attachment): void
{
$disk = Storage::disk('upload');
try {
$content = $disk->get(sprintf('at-%d.data', $attachment->id));
$content = Crypt::decrypt($content);
} catch (FileNotFoundException $e) {
Log::error($e->getMessage());
throw new FireflyException($e->getMessage());
}
$json = json_decode($content, true);
if (null !== $json) {
$this->repository->setConfiguration($this->importJob, $json);
}
}
}