. */ declare(strict_types=1); namespace FireflyIII\Repositories\ImportJob; use Crypt; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\Attachment; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Tag; use FireflyIII\Models\TransactionJournalMeta; use FireflyIII\Repositories\User\UserRepositoryInterface; use FireflyIII\User; use Illuminate\Support\Collection; use Illuminate\Support\MessageBag; use Illuminate\Support\Str; use Log; use SplFileObject; use Storage; use Symfony\Component\HttpFoundation\File\UploadedFile; /** * Class ImportJobRepository. */ class ImportJobRepository implements ImportJobRepositoryInterface { /** @var \Illuminate\Contracts\Filesystem\Filesystem */ protected $uploadDisk; /** @var int */ private $maxUploadSize; /** @var User */ private $user; public function __construct() { $this->maxUploadSize = (int)config('firefly.maxUploadSize'); $this->uploadDisk = Storage::disk('upload'); } /** * @param ImportJob $job * @param int $index * @param string $error * * @return ImportJob */ public function addError(ImportJob $job, int $index, string $error): ImportJob { $extended = $this->getExtendedStatus($job); $extended['errors'][$index][] = $error; return $this->setExtendedStatus($job, $extended); } /** * Add message to job. * * @param ImportJob $job * @param string $error * * @return ImportJob */ public function addErrorMessage(ImportJob $job, string $error): ImportJob { $errors = $job->errors; $errors[] = $error; $job->errors = $errors; $job->save(); return $job; } /** * @param ImportJob $job * @param int $steps * * @return ImportJob */ public function addStepsDone(ImportJob $job, int $steps = 1): ImportJob { $status = $this->getExtendedStatus($job); $status['done'] += $steps; Log::debug(sprintf('Add %d to steps done for job "%s" making steps done %d', $steps, $job->key, $status['done'])); return $this->setExtendedStatus($job, $status); } /** * @param ImportJob $job * @param int $steps * * @return ImportJob */ public function addTotalSteps(ImportJob $job, int $steps = 1): ImportJob { $extended = $this->getExtendedStatus($job); $total = (int)($extended['steps'] ?? 0); $total += $steps; $extended['steps'] = $total; return $this->setExtendedStatus($job, $extended); } /** * Return number of imported rows with this hash value. * * @param string $hash * * @return int */ public function countByHash(string $hash): int { $json = json_encode($hash); $count = TransactionJournalMeta::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id') ->where('data', $json) ->where('name', 'importHash') ->count(); return (int)$count; } /** * @param string $importProvider * * @return ImportJob * * @throws FireflyException */ public function create(string $importProvider): ImportJob { $count = 0; $importProvider = strtolower($importProvider); while ($count < 30) { $key = Str::random(12); $existing = $this->findByKey($key); if (null === $existing->id) { $importJob = ImportJob::create( [ 'user_id' => $this->user->id, 'tag_id' => null, 'provider' => $importProvider, 'file_type' => '', 'key' => Str::random(12), 'status' => 'new', 'stage' => 'new', 'configuration' => [], 'extended_status' => [], 'transactions' => [], 'errors' => [], ] ); // breaks the loop: return $importJob; } ++$count; } throw new FireflyException('Could not create an import job with a unique key after 30 tries.'); } /** * @param string $key * * @return ImportJob */ public function findByKey(string $key): ImportJob { /** @var ImportJob $result */ $result = $this->user->importJobs()->where('key', $key)->first(['import_jobs.*']); if (null === $result) { return new ImportJob; } return $result; } /** * Return all attachments for job. * * @param ImportJob $job * * @return Collection */ public function getAttachments(ImportJob $job): Collection { return $job->attachments()->get(); } /** * Return configuration of job. * * @param ImportJob $job * * @return array */ public function getConfiguration(ImportJob $job): array { $config = $job->configuration; if (\is_array($config)) { return $config; } return []; } /** * Return extended status of job. * * @param ImportJob $job * * @return array */ public function getExtendedStatus(ImportJob $job): array { $status = $job->extended_status; if (\is_array($status)) { return $status; } return []; } /** * @param ImportJob $job * * @return string */ public function getStatus(ImportJob $job): string { return $job->status; } /** * @param ImportJob $job * @param UploadedFile $file * * @return bool */ public function processConfiguration(ImportJob $job, UploadedFile $file): bool { /** @var UserRepositoryInterface $repository */ $repository = app(UserRepositoryInterface::class); // demo user's configuration upload is ignored completely. if (!$repository->hasRole($this->user, 'demo')) { Log::debug( 'Uploaded configuration file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()] ); $configFileObject = new SplFileObject($file->getRealPath()); $configRaw = $configFileObject->fread($configFileObject->getSize()); $configuration = json_decode($configRaw, true); Log::debug(sprintf('Raw configuration is %s', $configRaw)); if (null !== $configuration && \is_array($configuration)) { Log::debug('Found configuration', $configuration); $this->setConfiguration($job, $configuration); } if (null === $configuration) { Log::error('Uploaded configuration is NULL'); } if (false === $configuration) { Log::error('Uploaded configuration is FALSE'); } } return true; } /** * @param ImportJob $job * @param null|UploadedFile $file * * @return bool * * @throws \Illuminate\Contracts\Encryption\EncryptException * @throws \Illuminate\Contracts\Filesystem\FileNotFoundException */ public function processFile(ImportJob $job, ?UploadedFile $file): bool { if (null === $file) { return false; } /** @var UserRepositoryInterface $repository */ $repository = app(UserRepositoryInterface::class); $newName = sprintf('%s.upload', $job->key); $uploaded = new SplFileObject($file->getRealPath()); $content = trim($uploaded->fread($uploaded->getSize())); // verify content: $result = mb_detect_encoding($content, 'UTF-8', true); if ($result === false) { Log::error(sprintf('Cannot detect encoding for uploaded import file "%s".', $file->getClientOriginalName())); return false; } if ($result !== 'ASCII' && $result !== 'UTF-8') { Log::error(sprintf('Uploaded import file is %s instead of UTF8!', var_export($result, true))); return false; } $contentEncrypted = Crypt::encrypt($content); $disk = Storage::disk('upload'); // user is demo user, replace upload with prepared file. if ($repository->hasRole($this->user, 'demo')) { $stubsDisk = Storage::disk('stubs'); $content = $stubsDisk->get('demo-import.csv'); $contentEncrypted = Crypt::encrypt($content); $disk->put($newName, $contentEncrypted); Log::debug('Replaced upload with demo file.'); // also set up prepared configuration. $configuration = json_decode($stubsDisk->get('demo-configuration.json'), true); $this->setConfiguration($job, $configuration); Log::debug('Set configuration for demo user', $configuration); } if (!$repository->hasRole($this->user, 'demo')) { // user is not demo, process original upload: $disk->put($newName, $contentEncrypted); Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]); } return true; } /** * @param ImportJob $job * @param array $configuration * * @return ImportJob */ public function setConfiguration(ImportJob $job, array $configuration): ImportJob { Log::debug('Updating configuration...'); //Log::debug(sprintf('Incoming config for job "%s" is: ', $job->key), $configuration); $currentConfig = $job->configuration; $newConfig = array_merge($currentConfig, $configuration); $job->configuration = $newConfig; $job->save(); //Log::debug(sprintf('Set config of job "%s" to: ', $job->key), $newConfig); return $job; } /** * @param ImportJob $job * @param array $array * * @return ImportJob */ public function setExtendedStatus(ImportJob $job, array $array): ImportJob { $currentStatus = $job->extended_status; $newStatus = array_merge($currentStatus, $array); $job->extended_status = $newStatus; $job->save(); return $job; } /** * @param ImportJob $job * @param string $stage * * @return ImportJob */ public function setStage(ImportJob $job, string $stage): ImportJob { $job->stage = $stage; $job->save(); return $job; } /** * @param ImportJob $job * @param string $status * * @return ImportJob */ public function setStatus(ImportJob $job, string $status): ImportJob { Log::debug(sprintf('Set status of job "%s" to "%s"', $job->key, $status)); $job->status = $status; $job->save(); return $job; } /** * @param ImportJob $job * @param int $steps * * @return ImportJob */ public function setStepsDone(ImportJob $job, int $steps): ImportJob { $status = $this->getExtendedStatus($job); $status['done'] = $steps; Log::debug(sprintf('Set steps done for job "%s" to %d', $job->key, $steps)); return $this->setExtendedStatus($job, $status); } /** * @param ImportJob $job * @param Tag $tag * * @return ImportJob */ public function setTag(ImportJob $job, Tag $tag): ImportJob { $job->tag()->associate($tag); $job->save(); return $job; } /** * @param ImportJob $job * @param int $count * * @return ImportJob */ public function setTotalSteps(ImportJob $job, int $count): ImportJob { $status = $this->getExtendedStatus($job); $status['steps'] = $count; Log::debug(sprintf('Set total steps for job "%s" to %d', $job->key, $count)); return $this->setExtendedStatus($job, $status); } /** * @param ImportJob $job * @param array $transactions * * @return ImportJob */ public function setTransactions(ImportJob $job, array $transactions): ImportJob { $job->transactions = $transactions; $job->save(); return $job; } /** * @param User $user */ public function setUser(User $user) { $this->user = $user; } /** * Handle upload for job. * * @param ImportJob $job * @param string $name * @param UploadedFile $file * * @return MessageBag * @throws FireflyException */ public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag { $messages = new MessageBag; if (!file_exists($fileName)) { $messages->add('notfound', sprintf('File not found: %s', $fileName)); return $messages; } $count = $job->attachments()->get()->filter( function (Attachment $att) use ($name) { return $att->filename === $name; } )->count(); if ($count > 0) { // don't upload, but also don't complain about it. Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name)); return new MessageBag; } $content = file_get_contents($fileName); $attachment = new Attachment; // create Attachment object. $attachment->user()->associate($job->user); $attachment->attachable()->associate($job); $attachment->md5 = md5($content); $attachment->filename = $name; $attachment->mime = 'plain/txt'; $attachment->size = \strlen($content); $attachment->uploaded = 0; $attachment->save(); $encrypted = Crypt::encrypt($content); // store it: $this->uploadDisk->put($attachment->fileName(), $encrypted); $attachment->uploaded = 1; // update attachment $attachment->save(); // return it. return new MessageBag; } /** * Handle upload for job. * * @param ImportJob $job * @param string $name * @param UploadedFile $file * * @return MessageBag * @throws FireflyException */ public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag { $messages = new MessageBag; if ($this->validSize($file)) { $name = e($file->getClientOriginalName()); $messages->add('size', (string)trans('validation.file_too_large', ['name' => $name])); return $messages; } $count = $job->attachments()->get()->filter( function (Attachment $att) use ($name) { return $att->filename === $name; } )->count(); if ($count > 0) { // don't upload, but also don't complain about it. Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name)); return new MessageBag; } $attachment = new Attachment; // create Attachment object. $attachment->user()->associate($job->user); $attachment->attachable()->associate($job); $attachment->md5 = md5_file($file->getRealPath()); $attachment->filename = $name; $attachment->mime = $file->getMimeType(); $attachment->size = $file->getSize(); $attachment->uploaded = 0; $attachment->save(); $fileObject = $file->openFile('r'); $fileObject->rewind(); $content = $fileObject->fread($file->getSize()); $encrypted = Crypt::encrypt($content); // store it: $this->uploadDisk->put($attachment->fileName(), $encrypted); $attachment->uploaded = 1; // update attachment $attachment->save(); // return it. return new MessageBag; } /** * @param ImportJob $job * @param string $status * * @return ImportJob */ public function updateStatus(ImportJob $job, string $status): ImportJob { $job->status = $status; $job->save(); return $job; } /** * Return import file content. * * @deprecated * * @param ImportJob $job * * @return string * @throws \Illuminate\Contracts\Filesystem\FileNotFoundException */ public function uploadFileContents(ImportJob $job): string { return $job->uploadFileContents(); } /** * @codeCoverageIgnore * * @param UploadedFile $file * * @return bool */ protected function validSize(UploadedFile $file): bool { $size = $file->getSize(); return $size > $this->maxUploadSize; } }