Merge pull request #1267 from acelaya-forks/feature/rabbitmq

Feature/rabbitmq
This commit is contained in:
Alejandro Celaya 2021-12-12 11:43:43 +01:00 committed by GitHub
commit 2aec759857
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 400 additions and 21 deletions

View File

@ -14,6 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
* The `GET /domains` endpoint includes a new `defaultRedirects` property in the response, with the default redirects set via config or env vars.
* The `INVALID_SHORT_URL_REDIRECT_TO`, `REGULAR_404_REDIRECT_TO` and `BASE_URL_REDIRECT_TO` env vars are now deprecated, and should be replaced by `DEFAULT_INVALID_SHORT_URL_REDIRECT`, `DEFAULT_REGULAR_404_REDIRECT` and `DEFAULT_BASE_URL_REDIRECT` respectively. Deprecated ones will continue to work until v3.0.0, where they will be removed.
* [#868](https://github.com/shlinkio/shlink/issues/868) Added support to publish real-time updates in a RabbitMQ server.
Shlink will create new exchanges and queues for every topic documented in the [Async API spec](https://api-spec.shlink.io/async-api/), meaning, you will have one queue for orphan visits, one for regular visits, and one queue for every short URL with its visits.
The RabbitMQ server config can be provided via installer config options, or via environment variables.
* [#1204](https://github.com/shlinkio/shlink/issues/1204) Added support for `openswoole` and migrated official docker image to `openswoole`.
* [#1242](https://github.com/shlinkio/shlink/issues/1242) Added support to import urls and visits from YOURLS.
@ -29,7 +35,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
* [#1218](https://github.com/shlinkio/shlink/issues/1218) Updated to symfony/mercure 0.6.
* [#1223](https://github.com/shlinkio/shlink/issues/1223) Updated to phpstan 1.0.
* Added `domain` field to `DeleteShortUrlException` exception.
* [#1001](https://github.com/shlinkio/shlink/issues/1001) Increased required MSI to 83%.
### Deprecated
* [#1260](https://github.com/shlinkio/shlink/issues/1260) Deprecated `USE_HTTPS` env var that was added in previous release, in favor of the new `IS_HTTPS_ENABLED`.

View File

@ -11,8 +11,8 @@ WORKDIR /etc/shlink
# Install required PHP extensions
RUN \
# Install mysql and calendar
docker-php-ext-install -j"$(nproc)" pdo_mysql calendar && \
# Install extensions with no extra dependencies
docker-php-ext-install -j"$(nproc)" pdo_mysql calendar sockets bcmath && \
# Install sqlite
apk add --no-cache sqlite-libs sqlite-dev && \
docker-php-ext-install -j"$(nproc)" pdo_sqlite && \

View File

@ -34,10 +34,11 @@ The idea is that you can just generate a container using the image and provide t
First, make sure the host where you are going to run shlink fulfills these requirements:
* PHP 8.0
* PHP 8.0 or 8.1
* The next PHP extensions: json, curl, pdo, intl, gd and gmp.
* apcu extension is recommended if you don't plan to use swoole or openswoole.
* xml extension is required if you want to generate QR codes in svg format.
* sockets and bcmath extensions are required if you want to integrate with a RabbitMQ instance.
* MySQL, MariaDB, PostgreSQL, Microsoft SQL Server or SQLite.
* The web server of your choice with PHP integration (Apache or Nginx recommended).

View File

@ -42,16 +42,17 @@
"nikolaposa/monolog-factory": "^3.1",
"ocramius/proxy-manager": "^2.11",
"pagerfanta/core": "^3.5",
"php-amqplib/php-amqplib": "^3.1",
"php-middleware/request-id": "^4.1",
"predis/predis": "^1.1",
"pugx/shortid-php": "^1.0",
"ramsey/uuid": "^4.2",
"rlanvin/php-ip": "dev-master#6b3a785 as 3.0",
"shlinkio/shlink-common": "dev-main#c2e3442 as 4.2",
"shlinkio/shlink-common": "dev-main#e7fdff3 as 4.2",
"shlinkio/shlink-config": "^1.4",
"shlinkio/shlink-event-dispatcher": "dev-main#3925299 as 2.3",
"shlinkio/shlink-importer": "dev-main#d099072 as 2.5",
"shlinkio/shlink-installer": "dev-develop#7dd00fb as 6.3",
"shlinkio/shlink-installer": "^6.3",
"shlinkio/shlink-ip-geolocation": "^2.2",
"symfony/console": "^6.0 || ^5.4",
"symfony/filesystem": "^6.0 || ^5.4",
@ -141,7 +142,7 @@
"test:api": "bin/test/run-api-tests.sh",
"test:api:ci": "GENERATE_COVERAGE=yes composer test:api",
"infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --only-covering-test-cases --skip-initial-tests",
"infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=83",
"infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=80",
"infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json",
"infect:ci:api": "@infect:ci:base --coverage=build/coverage-api --min-msi=80 --configuration=infection-api.json",
"infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api",

View File

@ -57,6 +57,12 @@ return [
Option\QrCode\DefaultFormatConfigOption::class,
Option\QrCode\DefaultErrorCorrectionConfigOption::class,
Option\QrCode\DefaultRoundBlockSizeConfigOption::class,
Option\RabbitMq\RabbitMqEnabledConfigOption::class,
Option\RabbitMq\RabbitMqHostConfigOption::class,
Option\RabbitMq\RabbitMqPortConfigOption::class,
Option\RabbitMq\RabbitMqUserConfigOption::class,
Option\RabbitMq\RabbitMqPasswordConfigOption::class,
Option\RabbitMq\RabbitMqVhostConfigOption::class,
],
'installation_commands' => [

View File

@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory;
use Laminas\ServiceManager\Proxy\LazyServiceFactory;
use PhpAmqpLib\Connection\AMQPStreamConnection;
use function Shlinkio\Shlink\Common\env;
return [
'rabbitmq' => [
'enabled' => (bool) env('RABBITMQ_ENABLED', false),
'host' => env('RABBITMQ_HOST'),
'port' => (int) env('RABBITMQ_PORT', '5672'),
'user' => env('RABBITMQ_USER'),
'password' => env('RABBITMQ_PASSWORD'),
'vhost' => env('RABBITMQ_VHOST', '/'),
],
'dependencies' => [
'factories' => [
AMQPStreamConnection::class => ConfigAbstractFactory::class,
],
'delegators' => [
AMQPStreamConnection::class => [
LazyServiceFactory::class,
],
],
'lazy_services' => [
'class_map' => [
AMQPStreamConnection::class => AMQPStreamConnection::class,
],
],
],
ConfigAbstractFactory::class => [
AMQPStreamConnection::class => [
'config.rabbitmq.host',
'config.rabbitmq.port',
'config.rabbitmq.user',
'config.rabbitmq.password',
'config.rabbitmq.vhost',
],
],
];

View File

@ -0,0 +1,14 @@
<?php
declare(strict_types=1);
return [
'rabbitmq' => [
'enabled' => true,
'host' => 'shlink_rabbitmq',
'user' => 'rabbit',
'password' => 'rabbit',
],
];

View File

@ -34,6 +34,9 @@ RUN docker-php-ext-install pdo_pgsql
RUN apk add --no-cache gmp-dev
RUN docker-php-ext-install gmp
RUN docker-php-ext-install sockets
RUN docker-php-ext-install bcmath
# Install APCu extension
ADD https://pecl.php.net/get/apcu-$APCU_VERSION.tgz /tmp/apcu.tar.gz
RUN mkdir -p /usr/src/php/ext/apcu \

View File

@ -36,6 +36,9 @@ RUN docker-php-ext-install pdo_pgsql
RUN apk add --no-cache gmp-dev
RUN docker-php-ext-install gmp
RUN docker-php-ext-install sockets
RUN docker-php-ext-install bcmath
# Install APCu extension
ADD https://pecl.php.net/get/apcu-$APCU_VERSION.tgz /tmp/apcu.tar.gz
RUN mkdir -p /usr/src/php/ext/apcu \

View File

@ -29,6 +29,7 @@ services:
- shlink_redis
- shlink_mercure
- shlink_mercure_proxy
- shlink_rabbitmq
environment:
LC_ALL: C
extra_hosts:
@ -64,6 +65,7 @@ services:
- shlink_redis
- shlink_mercure
- shlink_mercure_proxy
- shlink_rabbitmq
environment:
LC_ALL: C
extra_hosts:
@ -143,3 +145,13 @@ services:
MERCURE_PUBLISHER_JWT_KEY: mercure_jwt_key
MERCURE_SUBSCRIBER_JWT_KEY: mercure_jwt_key
MERCURE_EXTRA_DIRECTIVES: "cors_origins https://app.shlink.io http://localhost:3000 http://127.0.0.1:3000"
shlink_rabbitmq:
container_name: shlink_rabbitmq
image: rabbitmq:3.9-management-alpine
ports:
- "15672:15672"
- "5672:5672"
environment:
RABBITMQ_DEFAULT_USER: "rabbit"
RABBITMQ_DEFAULT_PASS: "rabbit"

View File

@ -11,7 +11,7 @@
},
"defaultContentType": "application/json",
"channels": {
"http://shlink.io/new-visit": {
"https://shlink.io/new-visit": {
"subscribe": {
"summary": "Receive information about any new visit occurring on any short URL.",
"operationId": "newVisit",
@ -31,7 +31,7 @@
}
}
},
"http://shlink.io/new-visit/{shortCode}": {
"https://shlink.io/new-visit/{shortCode}": {
"parameters": {
"shortCode": {
"description": "The short code of the short URL",
@ -59,7 +59,7 @@
}
}
},
"http://shlink.io/new-orphan-visit": {
"https://shlink.io/new-orphan-visit": {
"subscribe": {
"summary": "Receive information about any new orphan visit.",
"operationId": "newOrphanVisit",

View File

@ -5,6 +5,7 @@ declare(strict_types=1);
namespace Shlinkio\Shlink\Core;
use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory;
use PhpAmqpLib\Connection\AMQPStreamConnection;
use Psr\EventDispatcher\EventDispatcherInterface;
use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater;
use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater;
@ -22,6 +23,7 @@ return [
'async' => [
EventDispatcher\Event\VisitLocated::class => [
EventDispatcher\NotifyVisitToMercure::class,
EventDispatcher\NotifyVisitToRabbitMq::class,
EventDispatcher\NotifyVisitToWebHooks::class,
EventDispatcher\UpdateGeoLiteDb::class,
],
@ -33,6 +35,7 @@ return [
EventDispatcher\LocateVisit::class => ConfigAbstractFactory::class,
EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class,
EventDispatcher\NotifyVisitToMercure::class => ConfigAbstractFactory::class,
EventDispatcher\NotifyVisitToRabbitMq::class => ConfigAbstractFactory::class,
EventDispatcher\UpdateGeoLiteDb::class => ConfigAbstractFactory::class,
],
@ -40,6 +43,9 @@ return [
EventDispatcher\NotifyVisitToMercure::class => [
EventDispatcher\CloseDbConnectionEventListenerDelegator::class,
],
EventDispatcher\NotifyVisitToRabbitMq::class => [
EventDispatcher\CloseDbConnectionEventListenerDelegator::class,
],
EventDispatcher\NotifyVisitToWebHooks::class => [
EventDispatcher\CloseDbConnectionEventListenerDelegator::class,
],
@ -68,6 +74,13 @@ return [
'em',
'Logger_Shlink',
],
EventDispatcher\NotifyVisitToRabbitMq::class => [
AMQPStreamConnection::class,
'em',
'Logger_Shlink',
Visit\Transformer\OrphanVisitDataTransformer::class,
'config.rabbitmq.enabled',
],
EventDispatcher\UpdateGeoLiteDb::class => [GeolocationDbUpdater::class, 'Logger_Shlink'],
],

View File

@ -0,0 +1,102 @@
<?php
declare(strict_types=1);
namespace Shlinkio\Shlink\Core\EventDispatcher;
use Doctrine\ORM\EntityManagerInterface;
use PhpAmqpLib\Connection\AMQPStreamConnection;
use PhpAmqpLib\Exchange\AMQPExchangeType;
use PhpAmqpLib\Message\AMQPMessage;
use Psr\Log\LoggerInterface;
use Shlinkio\Shlink\Common\Rest\DataTransformerInterface;
use Shlinkio\Shlink\Core\Entity\Visit;
use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated;
use Throwable;
use function Shlinkio\Shlink\Common\json_encode;
use function sprintf;
class NotifyVisitToRabbitMq
{
private const NEW_VISIT_QUEUE = 'https://shlink.io/new-visit';
private const NEW_ORPHAN_VISIT_QUEUE = 'https://shlink.io/new-orphan-visit';
public function __construct(
private AMQPStreamConnection $connection,
private EntityManagerInterface $em,
private LoggerInterface $logger,
private DataTransformerInterface $orphanVisitTransformer,
private bool $isEnabled,
) {
}
public function __invoke(VisitLocated $shortUrlLocated): void
{
if (! $this->isEnabled) {
return;
}
$visitId = $shortUrlLocated->visitId();
$visit = $this->em->find(Visit::class, $visitId);
if ($visit === null) {
$this->logger->warning('Tried to notify RabbitMQ for visit with id "{visitId}", but it does not exist.', [
'visitId' => $visitId,
]);
return;
}
if (! $this->connection->isConnected()) {
$this->connection->reconnect();
}
$queues = $this->determineQueuesToPublishTo($visit);
$message = $this->visitToMessage($visit);
try {
$channel = $this->connection->channel();
foreach ($queues as $queue) {
// Declare an exchange and a queue that will persist server restarts
$exchange = $queue; // We use the same name for the exchange and the queue
$channel->exchange_declare($exchange, AMQPExchangeType::DIRECT, false, true, false);
$channel->queue_declare($queue, false, true, false, false);
// Bind the exchange and the queue together, and publish the message
$channel->queue_bind($queue, $exchange);
$channel->basic_publish($message, $exchange);
}
$channel->close();
} catch (Throwable $e) {
$this->logger->debug('Error while trying to notify RabbitMQ with new visit. {e}', ['e' => $e]);
} finally {
$this->connection->close();
}
}
/**
* @return string[]
*/
private function determineQueuesToPublishTo(Visit $visit): array
{
if ($visit->isOrphan()) {
return [self::NEW_ORPHAN_VISIT_QUEUE];
}
return [
self::NEW_VISIT_QUEUE,
sprintf('%s/%s', self::NEW_VISIT_QUEUE, $visit->getShortUrl()?->getShortCode()),
];
}
private function visitToMessage(Visit $visit): AMQPMessage
{
$messageBody = json_encode(! $visit->isOrphan() ? $visit : $this->orphanVisitTransformer->transform($visit));
return new AMQPMessage($messageBody, [
'content_type' => 'application/json',
'delivery_mode' => AMQPMessage::DELIVERY_MODE_PERSISTENT,
]);
}
}

View File

@ -8,11 +8,9 @@ use Shlinkio\Shlink\Common\Rest\DataTransformerInterface;
use Shlinkio\Shlink\Core\Entity\Visit;
use Symfony\Component\Mercure\Update;
use function json_encode;
use function Shlinkio\Shlink\Common\json_encode;
use function sprintf;
use const JSON_THROW_ON_ERROR;
final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface
{
private const NEW_VISIT_TOPIC = 'https://shlink.io/new-visit';
@ -26,7 +24,7 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface
public function newVisitUpdate(Visit $visit): Update
{
return new Update(self::NEW_VISIT_TOPIC, $this->serialize([
return new Update(self::NEW_VISIT_TOPIC, json_encode([
'shortUrl' => $this->shortUrlTransformer->transform($visit->getShortUrl()),
'visit' => $visit,
]));
@ -34,7 +32,7 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface
public function newOrphanVisitUpdate(Visit $visit): Update
{
return new Update(self::NEW_ORPHAN_VISIT_TOPIC, $this->serialize([
return new Update(self::NEW_ORPHAN_VISIT_TOPIC, json_encode([
'visit' => $this->orphanVisitTransformer->transform($visit),
]));
}
@ -44,14 +42,9 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface
$shortUrl = $visit->getShortUrl();
$topic = sprintf('%s/%s', self::NEW_VISIT_TOPIC, $shortUrl?->getShortCode());
return new Update($topic, $this->serialize([
return new Update($topic, json_encode([
'shortUrl' => $this->shortUrlTransformer->transform($shortUrl),
'visit' => $visit,
]));
}
private function serialize(array $data): string
{
return json_encode($data, JSON_THROW_ON_ERROR);
}
}

View File

@ -0,0 +1,178 @@
<?php
declare(strict_types=1);
namespace ShlinkioTest\Shlink\Core\EventDispatcher;
use Doctrine\ORM\EntityManagerInterface;
use DomainException;
use Exception;
use PhpAmqpLib\Channel\AMQPChannel;
use PhpAmqpLib\Connection\AMQPStreamConnection;
use PHPUnit\Framework\TestCase;
use Prophecy\Argument;
use Prophecy\PhpUnit\ProphecyTrait;
use Prophecy\Prophecy\ObjectProphecy;
use Psr\Log\LoggerInterface;
use RuntimeException;
use Shlinkio\Shlink\Core\Entity\ShortUrl;
use Shlinkio\Shlink\Core\Entity\Visit;
use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated;
use Shlinkio\Shlink\Core\EventDispatcher\NotifyVisitToRabbitMq;
use Shlinkio\Shlink\Core\Model\ShortUrlMeta;
use Shlinkio\Shlink\Core\Model\Visitor;
use Shlinkio\Shlink\Core\Visit\Transformer\OrphanVisitDataTransformer;
use Throwable;
use function count;
use function Functional\contains;
class NotifyVisitToRabbitMqTest extends TestCase
{
use ProphecyTrait;
private NotifyVisitToRabbitMq $listener;
private ObjectProphecy $connection;
private ObjectProphecy $em;
private ObjectProphecy $logger;
private ObjectProphecy $orphanVisitTransformer;
private ObjectProphecy $channel;
protected function setUp(): void
{
$this->channel = $this->prophesize(AMQPChannel::class);
$this->connection = $this->prophesize(AMQPStreamConnection::class);
$this->connection->isConnected()->willReturn(false);
$this->connection->channel()->willReturn($this->channel->reveal());
$this->em = $this->prophesize(EntityManagerInterface::class);
$this->logger = $this->prophesize(LoggerInterface::class);
$this->listener = new NotifyVisitToRabbitMq(
$this->connection->reveal(),
$this->em->reveal(),
$this->logger->reveal(),
new OrphanVisitDataTransformer(),
true,
);
}
/** @test */
public function doesNothingWhenTheFeatureIsNotEnabled(): void
{
$listener = new NotifyVisitToRabbitMq(
$this->connection->reveal(),
$this->em->reveal(),
$this->logger->reveal(),
new OrphanVisitDataTransformer(),
false,
);
$listener(new VisitLocated('123'));
$this->em->find(Argument::cetera())->shouldNotHaveBeenCalled();
$this->logger->warning(Argument::cetera())->shouldNotHaveBeenCalled();
$this->logger->debug(Argument::cetera())->shouldNotHaveBeenCalled();
$this->connection->isConnected()->shouldNotHaveBeenCalled();
$this->connection->close()->shouldNotHaveBeenCalled();
}
/** @test */
public function notificationsAreNotSentWhenVisitCannotBeFound(): void
{
$visitId = '123';
$findVisit = $this->em->find(Visit::class, $visitId)->willReturn(null);
$logWarning = $this->logger->warning(
'Tried to notify RabbitMQ for visit with id "{visitId}", but it does not exist.',
['visitId' => $visitId],
);
($this->listener)(new VisitLocated($visitId));
$findVisit->shouldHaveBeenCalledOnce();
$logWarning->shouldHaveBeenCalledOnce();
$this->logger->debug(Argument::cetera())->shouldNotHaveBeenCalled();
$this->connection->isConnected()->shouldNotHaveBeenCalled();
$this->connection->close()->shouldNotHaveBeenCalled();
}
/**
* @test
* @dataProvider provideVisits
*/
public function expectedChannelsAreNotifiedBasedOnTheVisitType(Visit $visit, array $expectedChannels): void
{
$visitId = '123';
$findVisit = $this->em->find(Visit::class, $visitId)->willReturn($visit);
$argumentWithExpectedChannel = Argument::that(fn (string $channel) => contains($expectedChannels, $channel));
($this->listener)(new VisitLocated($visitId));
$findVisit->shouldHaveBeenCalledOnce();
$this->channel->exchange_declare($argumentWithExpectedChannel, Argument::cetera())->shouldHaveBeenCalledTimes(
count($expectedChannels),
);
$this->channel->queue_declare($argumentWithExpectedChannel, Argument::cetera())->shouldHaveBeenCalledTimes(
count($expectedChannels),
);
$this->channel->queue_bind(
$argumentWithExpectedChannel,
$argumentWithExpectedChannel,
)->shouldHaveBeenCalledTimes(count($expectedChannels));
$this->channel->basic_publish(Argument::any(), $argumentWithExpectedChannel)->shouldHaveBeenCalledTimes(
count($expectedChannels),
);
$this->channel->close()->shouldHaveBeenCalledOnce();
$this->connection->reconnect()->shouldHaveBeenCalledOnce();
$this->connection->close()->shouldHaveBeenCalledOnce();
$this->logger->debug(Argument::cetera())->shouldNotHaveBeenCalled();
}
public function provideVisits(): iterable
{
$visitor = Visitor::emptyInstance();
yield 'orphan visit' => [Visit::forBasePath($visitor), ['https://shlink.io/new-orphan-visit']];
yield 'non-orphan visit' => [
Visit::forValidShortUrl(
ShortUrl::fromMeta(ShortUrlMeta::fromRawData([
'longUrl' => 'foo',
'customSlug' => 'bar',
])),
$visitor,
),
['https://shlink.io/new-visit', 'https://shlink.io/new-visit/bar'],
];
}
/**
* @test
* @dataProvider provideExceptions
*/
public function printsDebugMessageInCaseOfError(Throwable $e): void
{
$visitId = '123';
$findVisit = $this->em->find(Visit::class, $visitId)->willReturn(Visit::forBasePath(Visitor::emptyInstance()));
$channel = $this->connection->channel()->willThrow($e);
($this->listener)(new VisitLocated($visitId));
$this->logger->debug(
'Error while trying to notify RabbitMQ with new visit. {e}',
['e' => $e],
)->shouldHaveBeenCalledOnce();
$this->connection->close()->shouldHaveBeenCalledOnce();
$this->connection->reconnect()->shouldHaveBeenCalledOnce();
$findVisit->shouldHaveBeenCalledOnce();
$channel->shouldHaveBeenCalledOnce();
$this->channel->close()->shouldNotHaveBeenCalled();
}
public function provideExceptions(): iterable
{
yield [new RuntimeException('RuntimeException Error')];
yield [new Exception('Exception Error')];
yield [new DomainException('DomainException Error')];
}
}