From 33d3837795a08f31a5882efc4e9decda228d4e21 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 22 Oct 2020 18:12:22 +0200 Subject: [PATCH] Added dependency on shlinkio/shlink-importer --- CHANGELOG.md | 1 + composer.json | 1 + config/autoload/dependencies.global.php | 6 ++ config/config.php | 1 + module/Core/config/dependencies.config.php | 9 +++ module/Core/src/Entity/ShortUrl.php | 24 ++++++++ .../src/Importer/ImportedLinksProcessor.php | 55 +++++++++++++++++++ 7 files changed, 97 insertions(+) create mode 100644 module/Core/src/Importer/ImportedLinksProcessor.php diff --git a/CHANGELOG.md b/CHANGELOG.md index ff32f795..7c044b9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * [#837](https://github.com/shlinkio/shlink/issues/837) Drastically improved performance when creating a new shortUrl and providing `findIfExists = true`. + ## 2.3.0 - 2020-08-09 #### Added diff --git a/composer.json b/composer.json index 99844320..597cc4b1 100644 --- a/composer.json +++ b/composer.json @@ -53,6 +53,7 @@ "shlinkio/shlink-common": "^3.2.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", + "shlinkio/shlink-importer": "^1.0", "shlinkio/shlink-installer": "^5.1.0", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", diff --git a/config/autoload/dependencies.global.php b/config/autoload/dependencies.global.php index 023b3c4e..dbc553f1 100644 --- a/config/autoload/dependencies.global.php +++ b/config/autoload/dependencies.global.php @@ -2,7 +2,9 @@ declare(strict_types=1); +use GuzzleHttp\Client; use Mezzio\Container; +use Psr\Http\Client\ClientInterface; return [ @@ -13,6 +15,10 @@ return [ ], ], + 'aliases' => [ + ClientInterface::class => Client::class, + ], + 'lazy_services' => [ 'proxies_target_dir' => 'data/proxies', 'proxies_namespace' => 'ShlinkProxy', diff --git a/config/config.php b/config/config.php index d7fd6a3b..ba0657fc 100644 --- a/config/config.php +++ b/config/config.php @@ -21,6 +21,7 @@ return (new ConfigAggregator\ConfigAggregator([ Diactoros\ConfigProvider::class, Common\ConfigProvider::class, Config\ConfigProvider::class, + Importer\ConfigProvider::class, IpGeolocation\ConfigProvider::class, EventDispatcher\ConfigProvider::class, Core\ConfigProvider::class, diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 5dcef9a2..09c4d96e 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -10,6 +10,7 @@ use Psr\EventDispatcher\EventDispatcherInterface; use Shlinkio\Shlink\Core\Domain\Resolver; use Shlinkio\Shlink\Core\ErrorHandler; use Shlinkio\Shlink\Core\Options\NotFoundRedirectOptions; +use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface; return [ @@ -42,6 +43,12 @@ return [ Resolver\PersistenceDomainResolver::class => ConfigAbstractFactory::class, Mercure\MercureUpdatesGenerator::class => ConfigAbstractFactory::class, + + Importer\ImportedLinksProcessor::class => ConfigAbstractFactory::class, + ], + + 'aliases' => [ + ImportedLinksProcessorInterface::class => Importer\ImportedLinksProcessor::class, ], ], @@ -96,6 +103,8 @@ return [ Resolver\PersistenceDomainResolver::class => ['em'], Mercure\MercureUpdatesGenerator::class => ['config.url_shortener.domain'], + + Importer\ImportedLinksProcessor::class => ['em', Resolver\PersistenceDomainResolver::class], ], ]; diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php index ba10a44a..6da6562a 100644 --- a/module/Core/src/Entity/ShortUrl.php +++ b/module/Core/src/Entity/ShortUrl.php @@ -14,6 +14,8 @@ use Shlinkio\Shlink\Core\Domain\Resolver\SimpleDomainResolver; use Shlinkio\Shlink\Core\Exception\ShortCodeCannotBeRegeneratedException; use Shlinkio\Shlink\Core\Model\ShortUrlEdit; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; +use Shlinkio\Shlink\Core\Validation\ShortUrlMetaInputFilter; +use Shlinkio\Shlink\Importer\Model\ShlinkUrl; use function count; use function Shlinkio\Shlink\Core\generateRandomShortCode; @@ -33,6 +35,7 @@ class ShortUrl extends AbstractEntity private ?Domain $domain = null; private bool $customSlugWasProvided; private int $shortCodeLength; + private ?string $source = null; public function __construct( string $longUrl, @@ -54,6 +57,27 @@ class ShortUrl extends AbstractEntity $this->domain = ($domainResolver ?? new SimpleDomainResolver())->resolveDomain($meta->getDomain()); } + public static function fromImport( + ShlinkUrl $url, + string $source, + bool $importShortCode, + ?DomainResolverInterface $domainResolver = null + ): self { + $meta = [ + ShortUrlMetaInputFilter::DOMAIN => $url->domain(), + ShortUrlMetaInputFilter::VALIDATE_URL => false, + ]; + if ($importShortCode) { + $meta[ShortUrlMetaInputFilter::CUSTOM_SLUG] = $url->shortCode(); + } + + $instance = new self($url->longUrl(), ShortUrlMeta::fromRawData($meta), $domainResolver); + $instance->source = $source; + $instance->dateCreated = Chronos::instance($url->createdAt()); + + return $instance; + } + public function getLongUrl(): string { return $this->longUrl; diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php new file mode 100644 index 00000000..cb3ca57b --- /dev/null +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -0,0 +1,55 @@ +em = $em; + $this->domainResolver = $domainResolver; + } + + /** + * @param ShlinkUrl[] $shlinkUrls + */ + public function process(iterable $shlinkUrls, string $source, array $params): void + { + $importShortCodes = $params['import_short_codes']; + $count = 0; + $persistBlock = 100; + + foreach ($shlinkUrls as $url) { + $count++; + + $shortUrl = ShortUrl::fromImport($url, $source, $importShortCodes, $this->domainResolver); + $shortUrl->setTags($this->tagNamesToEntities($this->em, $url->tags())); + + // TODO Handle errors while creating short URLs, to avoid making the whole process fail + $this->em->persist($shortUrl); + + // Flush and clear after X iterations + if ($count % $persistBlock === 0) { + $this->em->flush(); + $this->em->clear(); + } + } + + $this->em->flush(); + $this->em->clear(); + } +}